From d36674170515f849e96b4116ad1432f02a32d300 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 10 Jul 2023 21:38:03 +0200 Subject: [PATCH 001/148] add extensions --- .../TransactionExtensions.cs | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs index 47f20554226..dc5b0eb3603 100644 --- a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs @@ -62,5 +62,24 @@ internal static UInt256 CalculateAffordableGasPrice(this Transaction tx, bool ei return balance <= tx.Value ? default : tx.GasPrice; } + + internal static bool IsOverflowWhenAddingTxCostToCumulative(this Transaction tx, UInt256 currentCost, out UInt256 cumulativeCost) + { + bool overflow = false; + + overflow |= UInt256.MultiplyOverflow(tx.MaxFeePerGas, (UInt256)tx.GasLimit, out UInt256 maxTxCost); + overflow |= UInt256.AddOverflow(currentCost, maxTxCost, out cumulativeCost); + overflow |= UInt256.AddOverflow(cumulativeCost, tx.Value, out cumulativeCost); + + if (tx.SupportsBlobs) + { + // if tx.SupportsBlobs and has BlobVersionedHashes = null, it will throw on earlier step of validation, in TxValidator + overflow |= UInt256.MultiplyOverflow(Eip4844Constants.DataGasPerBlob, (UInt256)tx.BlobVersionedHashes!.Length, out UInt256 dataGas); + overflow |= UInt256.MultiplyOverflow(dataGas, tx.MaxFeePerDataGas ?? UInt256.MaxValue, out UInt256 dataGasCost); + overflow |= UInt256.AddOverflow(cumulativeCost, dataGasCost, out cumulativeCost); + } + + return overflow; + } } } From 53fcffacd21a011a082f2228b369000551c841be Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 10 Jul 2023 21:38:18 +0200 Subject: [PATCH 002/148] adjust filters --- .../Nethermind.TxPool/Filters/BalanceTooLowFilter.cs | 9 +++------ .../Nethermind.TxPool/Filters/BalanceZeroFilter.cs | 2 ++ .../Nethermind.TxPool/Filters/GapNonceFilter.cs | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs index 4d6806f3f67..1266c8c9af4 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs @@ -46,9 +46,7 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO if (otherTx.Nonce < tx.Nonce) { - overflow |= UInt256.MultiplyOverflow(otherTx.MaxFeePerGas, (UInt256)otherTx.GasLimit, out UInt256 maxTxCost); - overflow |= UInt256.AddOverflow(cumulativeCost, maxTxCost, out cumulativeCost); - overflow |= UInt256.AddOverflow(cumulativeCost, otherTx.Value, out cumulativeCost); + overflow |= otherTx.IsOverflowWhenAddingTxCostToCumulative(cumulativeCost, out cumulativeCost); } else { @@ -56,9 +54,8 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO } } - overflow |= UInt256.MultiplyOverflow(tx.MaxFeePerGas, (UInt256)tx.GasLimit, out UInt256 cost); - overflow |= UInt256.AddOverflow(cost, tx.Value, out cost); - overflow |= UInt256.AddOverflow(cost, cumulativeCost, out cumulativeCost); + overflow |= tx.IsOverflowWhenAddingTxCostToCumulative(cumulativeCost, out cumulativeCost); + if (overflow) { if (_logger.IsTrace) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs index b69eaa41a47..81fd6736444 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs @@ -43,6 +43,8 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO AcceptTxResult.InsufficientFunds.WithMessage($"Balance is {balance} less than sending value {tx.Value}"); } + // data gas cost is not added here, so in case of overflow at that step, blob transactions will be rejected + // later, in BalanceTooLowFilter. It's rare scenario, so probably it's not worth to complicate code here if (UInt256.MultiplyOverflow(tx.MaxFeePerGas, (UInt256)tx.GasLimit, out UInt256 txCostAndValue) || UInt256.AddOverflow(txCostAndValue, tx.Value, out txCostAndValue)) { diff --git a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs index 7c91adf19f8..283b386a455 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs @@ -26,7 +26,7 @@ public GapNonceFilter(TxDistinctSortedPool txs, ILogger logger) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; - if (isLocal || !_txs.IsFull()) + if ((isLocal || !_txs.IsFull()) && !tx.SupportsBlobs) { return AcceptTxResult.Accepted; } From 2bc9be502d20434b3e7d54ab4a1c2ba2980ac68a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 10 Jul 2023 21:38:25 +0200 Subject: [PATCH 003/148] add tests --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 75 ++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 0bf508f5719..b786eca0f22 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -60,6 +60,8 @@ public void Setup() Block block = Build.A.Block.WithNumber(0).TestObject; _blockTree.Head.Returns(block); _blockTree.FindBestSuggestedHeader().Returns(Build.A.BlockHeader.WithNumber(10000000).TestObject); + + KzgPolynomialCommitments.InitializeAsync().Wait(); } [Test] @@ -226,6 +228,14 @@ private static ISpecProvider GetLondonSpecProvider() return specProvider; } + private static ISpecProvider GetCancunSpecProvider() + { + var specProvider = Substitute.For(); + specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); + specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); + return specProvider; + } + [TestCase(false, false, ExpectedResult = nameof(AcceptTxResult.Accepted))] [TestCase(false, true, ExpectedResult = nameof(AcceptTxResult.Accepted))] [TestCase(true, false, ExpectedResult = nameof(AcceptTxResult.Accepted))] @@ -641,6 +651,37 @@ public void should_discard_tx_because_of_overflow_of_cumulative_cost_of_this_tx_ _txPool.SubmitTx(transactions[2], TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Int256Overflow); } + [Test] + public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, true)] bool supportsBlobs) + { + _txPool = CreatePool(null, GetCancunSpecProvider()); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + UInt256.MaxValue.Divide(GasCostOf.Transaction * 2, out UInt256 halfOfMaxGasPriceWithoutOverflow); + + Transaction firstTransaction = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(firstTransaction, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); + + Transaction transactionWithPotentialOverflow = Build.A.Transaction + .WithMaxFeePerDataGas(supportsBlobs + ? UInt256.One + : UInt256.Zero) // simplification - it's still blob tx, but if MaxDataFee is 0, cost for blob + // data is 0, so we are able to test overflow well as only this field differs + .WithNonce(UInt256.One) + .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithShardBlobTxTypeAndFields() + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(transactionWithPotentialOverflow, TxHandlingOptions.PersistentBroadcast).Should().Be(supportsBlobs ? AcceptTxResult.Int256Overflow : AcceptTxResult.Accepted); + } + [Test] public async Task should_not_dump_GasBottleneck_of_all_txs_in_bucket_if_first_tx_in_bucket_has_insufficient_balance_but_has_old_nonce() { @@ -1593,11 +1634,43 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); } + [Test] + public void should_add_nonce_gap_tx_to_not_full_TxPool_only_if_is_not_blob_type() + { + _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction nonceGapBlobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce((UInt256)2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction nonceGap1559Tx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce((UInt256)2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.NonceGap); + _txPool.SubmitTx(nonceGap1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] [TestCase(3, 393741)] - [TestCase(4, 524948)] + [TestCase(4, 524947)] [TestCase(5, 656156)] [TestCase(6, 787365)] public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int expectedLength) From 1a258858e3ad581b130d47e27f762f420039ab3b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 13 Jul 2023 21:41:57 +0200 Subject: [PATCH 004/148] draft --- .../TransactionSelectorTests.cs | 6 + .../Producers/TxPoolTxSource.cs | 91 +++--- .../TxBroadcasterTests.cs | 152 ++++----- .../Nethermind.TxPool.Test/TxPoolTests.cs | 299 +++++++++++++----- .../Nethermind.TxPool/AcceptTxResult.cs | 4 + .../Collections/BlobTxDistinctSortedPool.cs | 85 +++++ .../Collections/DistinctValueSortedPool.cs | 2 +- .../Collections/SortedPool.cs | 4 +- .../Collections/TxSortedPoolExtensions.cs | 3 + .../Comparison/CompareReplacedBlobTx.cs | 53 ++++ .../Filters/BalanceTooLowFilter.cs | 12 +- .../Filters/FeeTooLowFilter.cs | 2 +- .../Filters/GapNonceFilter.cs | 8 +- .../Filters/TxTypeTxFilter.cs | 34 ++ src/Nethermind/Nethermind.TxPool/ITxPool.cs | 8 +- src/Nethermind/Nethermind.TxPool/Metrics.cs | 12 + .../Nethermind.TxPool/NullTxPool.cs | 2 + .../Nethermind.TxPool/TxBroadcaster.cs | 3 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 76 +++-- 19 files changed, 630 insertions(+), 226 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs create mode 100644 src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs create mode 100644 src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index 2bdf521fd74..7faa0527bc5 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -233,11 +233,17 @@ void SetAccountStates(IEnumerable
missingAddresses) IComparer comparer = CompareTxByNonce.Instance.ThenBy(defaultComparer); Dictionary transactions = testCase.Transactions .Where(t => t?.SenderAddress is not null) + .Where(t => !t.SupportsBlobs) .GroupBy(t => t.SenderAddress) .ToDictionary( g => g.Key!, g => g.OrderBy(t => t, comparer).ToArray()); + Transaction[] blobTransactions = testCase.Transactions + .Where(t => t?.SenderAddress is not null) + .Where(t => t.SupportsBlobs) + .ToArray(); transactionPool.GetPendingTransactionsBySender().Returns(transactions); + transactionPool.GetBlobTransactions().Returns(blobTransactions); BlocksConfig blocksConfig = new() { MinGasPrice = testCase.MinGasPriceForMining }; ITxFilterPipeline txFilterPipeline = new TxFilterPipelineBuilder(LimboLogs.Instance) .WithMinGasPriceFilter(blocksConfig, specProvider) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index e5363d8237b..f7051478c70 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -53,71 +53,82 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi .ThenBy(ByHashTxComparer.Instance); // in order to sort properly and not lose transactions we need to differentiate on their identity which provided comparer might not be doing IEnumerable transactions = GetOrderedTransactions(pendingTransactions, comparer); + IEnumerable blobTransactions = _transactionPool.GetBlobTransactions(); if (_logger.IsDebug) _logger.Debug($"Collecting pending transactions at block gas limit {gasLimit}."); int selectedTransactions = 0; int i = 0; - - // TODO: removing transactions from TX pool here seems to be a bad practice since they will - // not come back if the block is ignored? int blobsCounter = 0; UInt256 dataGasPrice = UInt256.Zero; - foreach (Transaction tx in transactions) + foreach (Transaction blobTx in blobTransactions) { - i++; - - if (tx.SenderAddress is null) + if (DataGasCalculator.CalculateDataGas(blobsCounter) == Eip4844Constants.MaxDataGasPerBlock) { - _transactionPool.RemoveTransaction(tx.Hash!); - if (_logger.IsDebug) _logger.Debug($"Rejecting (null sender) {tx.ToShortString()}"); - continue; + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); + break; } - bool success = _txFilterPipeline.Execute(tx, parent); - if (!success) continue; + i++; - if (tx.SupportsBlobs) - { - if (dataGasPrice.IsZero) - { - ulong? excessDataGas = DataGasCalculator.CalculateExcessDataGas(parent, _specProvider.GetSpec(parent)); - if (excessDataGas is null) - { - if (_logger.IsTrace) _logger.Trace($"Declining {tx.ToShortString()}, the specification is not configured to handle shard blob transactions."); - continue; - } - if (!DataGasCalculator.TryCalculateDataGasPricePerUnit(excessDataGas.Value, out dataGasPrice)) - { - if (_logger.IsTrace) _logger.Trace($"Declining {tx.ToShortString()}, failed to calculate data gas price."); - continue; - } - } + //add removal of null sender txs? There shouldn't be null sender at this point - int txAmountOfBlobs = tx.BlobVersionedHashes?.Length ?? 0; + bool success = _txFilterPipeline.Execute(blobTx, parent); + if (!success) continue; - if (dataGasPrice > tx.MaxFeePerDataGas) + if (dataGasPrice.IsZero) + { + ulong? excessDataGas = DataGasCalculator.CalculateExcessDataGas(parent, _specProvider.GetSpec(parent)); + if (excessDataGas is null) { - if (_logger.IsTrace) _logger.Trace($"Declining {tx.ToShortString()}, data gas fee is too low."); + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); continue; } - - if (DataGasCalculator.CalculateDataGas(blobsCounter + txAmountOfBlobs) > - Eip4844Constants.MaxDataGasPerBlock) + if (!DataGasCalculator.TryCalculateDataGasPricePerUnit(excessDataGas.Value, out dataGasPrice)) { - if (_logger.IsTrace) _logger.Trace($"Declining {tx.ToShortString()}, no more blob space."); + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to calculate data gas price."); continue; } + } + + int txAmountOfBlobs = blobTx.BlobVersionedHashes?.Length ?? 0; + + if (dataGasPrice > blobTx.MaxFeePerDataGas) + { + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, data gas fee is too low."); + continue; + } - blobsCounter += txAmountOfBlobs; - if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {tx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); + if (DataGasCalculator.CalculateDataGas(blobsCounter + txAmountOfBlobs) > + Eip4844Constants.MaxDataGasPerBlock) + { + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, not enough blob space."); + continue; } - else + + blobsCounter += txAmountOfBlobs; + if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {blobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); + + selectedTransactions++; + yield return blobTx; + } + + foreach (Transaction tx in transactions) + { + i++; + + if (tx.SenderAddress is null) { - if (_logger.IsTrace) - _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); + _transactionPool.RemoveTransaction(tx.Hash!); + if (_logger.IsDebug) _logger.Debug($"Rejecting (null sender) {tx.ToShortString()}"); + continue; } + bool success = _txFilterPipeline.Execute(tx, parent); + if (!success) continue; + + if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); + selectedTransactions++; yield return tx; } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 67f76e7dfe2..ec55888799d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -153,81 +153,83 @@ public void should_pick_best_persistent_txs_to_broadcast(int threshold) expectedTxs.Should().BeEquivalentTo(pickedTxs); } - [TestCase(1)] - [TestCase(2)] - [TestCase(25)] - [TestCase(50)] - [TestCase(99)] - [TestCase(100)] - [TestCase(101)] - [TestCase(1000)] - public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast(int threshold) - { - _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; - _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); - _headInfo.CurrentBaseFee.Returns(0.GWei()); - - int addedTxsCount = TestItem.PrivateKeys.Length; - Transaction[] transactions = new Transaction[addedTxsCount]; - - for (int i = 0; i < addedTxsCount; i++) - { - bool isBlob = i % 10 == 0; - transactions[i] = Build.A.Transaction - .WithGasPrice((addedTxsCount - i - 1).GWei()) - .WithType(isBlob ? TxType.Blob : TxType.Legacy) //some part of txs (10%) is blob type - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) - .TestObject; - - _broadcaster.Broadcast(transactions[i], true); - } - - _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); - - (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); - - int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); - int expectedCountOfBlobHashes = expectedCountTotal / 10 + 1; - int expectedCountOfNonBlobTxs = expectedCountTotal - expectedCountOfBlobHashes; - if (expectedCountOfNonBlobTxs > 0) - { - pickedTxs.Count.Should().Be(expectedCountOfNonBlobTxs); - } - else - { - pickedTxs.Should().BeNull(); - } - - if (expectedCountOfBlobHashes > 0) - { - pickedHashes.Count.Should().Be(expectedCountOfBlobHashes); - } - else - { - pickedHashes.Should().BeNull(); - } - - List expectedTxs = new(); - List expectedHashes = new(); - - for (int i = 0; i < expectedCountTotal; i++) - { - Transaction tx = transactions[i]; - - if (!tx.SupportsBlobs) - { - expectedTxs.Add(tx); - } - else - { - expectedHashes.Add(tx); - } - } - - expectedTxs.Should().BeEquivalentTo(pickedTxs); - expectedHashes.Should().BeEquivalentTo(pickedHashes); - } + // commented out as for now we are not adding blob txs to persistent at all. + // work is still in progress so it might be back - keeping commented out for now + // [TestCase(1)] + // [TestCase(2)] + // [TestCase(25)] + // [TestCase(50)] + // [TestCase(99)] + // [TestCase(100)] + // [TestCase(101)] + // [TestCase(1000)] + // public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast(int threshold) + // { + // _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; + // _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + // _headInfo.CurrentBaseFee.Returns(0.GWei()); + // + // int addedTxsCount = TestItem.PrivateKeys.Length; + // Transaction[] transactions = new Transaction[addedTxsCount]; + // + // for (int i = 0; i < addedTxsCount; i++) + // { + // bool isBlob = i % 10 == 0; + // transactions[i] = Build.A.Transaction + // .WithGasPrice((addedTxsCount - i - 1).GWei()) + // .WithType(isBlob ? TxType.Blob : TxType.Legacy) //some part of txs (10%) is blob type + // .WithShardBlobTxTypeAndFieldsIfBlobTx() + // .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) + // .TestObject; + // + // _broadcaster.Broadcast(transactions[i], true); + // } + // + // _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); + // + // (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); + // + // int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); + // int expectedCountOfBlobHashes = expectedCountTotal / 10 + 1; + // int expectedCountOfNonBlobTxs = expectedCountTotal - expectedCountOfBlobHashes; + // if (expectedCountOfNonBlobTxs > 0) + // { + // pickedTxs.Count.Should().Be(expectedCountOfNonBlobTxs); + // } + // else + // { + // pickedTxs.Should().BeNull(); + // } + // + // if (expectedCountOfBlobHashes > 0) + // { + // pickedHashes.Count.Should().Be(expectedCountOfBlobHashes); + // } + // else + // { + // pickedHashes.Should().BeNull(); + // } + // + // List expectedTxs = new(); + // List expectedHashes = new(); + // + // for (int i = 0; i < expectedCountTotal; i++) + // { + // Transaction tx = transactions[i]; + // + // if (!tx.SupportsBlobs) + // { + // expectedTxs.Add(tx); + // } + // else + // { + // expectedHashes.Add(tx); + // } + // } + // + // expectedTxs.Should().BeEquivalentTo(pickedTxs); + // expectedHashes.Should().BeEquivalentTo(pickedHashes); + // } [TestCase(1)] [TestCase(2)] diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index b786eca0f22..7dc958e52de 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -2,7 +2,6 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; using System.Threading; @@ -100,7 +99,7 @@ public void should_ignore_transactions_with_different_chain_id() EthereumEcdsa ecdsa = new(BlockchainIds.Sepolia, _logManager); // default is mainnet, we're passing sepolia Transaction tx = Build.A.Transaction.SignedAndResolved(ecdsa, TestItem.PrivateKeyA).TestObject; AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.Invalid); } @@ -121,7 +120,7 @@ public void should_ignore_transactions_with_insufficient_intrinsic_gas() .TestObject; AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.Invalid); ; } @@ -132,7 +131,7 @@ public void should_not_ignore_old_scheme_signatures() Transaction tx = Build.A.Transaction.SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA, false).TestObject; EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); result.Should().Be(AcceptTxResult.Accepted); } @@ -144,7 +143,7 @@ public void should_ignore_already_known() EnsureSenderBalance(tx); AcceptTxResult result1 = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); AcceptTxResult result2 = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); result1.Should().Be(AcceptTxResult.Accepted); result2.Should().Be(AcceptTxResult.AlreadyKnown); } @@ -159,7 +158,7 @@ public void should_add_valid_transactions_recovering_its_address() EnsureSenderBalance(tx); tx.SenderAddress = null; AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); result.Should().Be(AcceptTxResult.Accepted); } @@ -196,7 +195,7 @@ public void should_accept_1559_transactions_only_when_eip1559_enabled([Values(fa EnsureSenderBalance(tx); _blockTree.BlockAddedToMain += Raise.EventWith(_blockTree, new BlockReplacementEventArgs(Build.A.Block.WithGasLimit(10000000).TestObject)); AcceptTxResult result = txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - txPool.GetPendingTransactions().Length.Should().Be(eip1559Enabled ? 1 : 0); + txPool.GetPendingTransactionsCount().Should().Be(eip1559Enabled ? 1 : 0); result.Should().Be(eip1559Enabled ? AcceptTxResult.Accepted : AcceptTxResult.Invalid); } @@ -211,29 +210,13 @@ public void should_not_ignore_insufficient_funds_for_eip1559_transactions() .WithValue(5).SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(tx.SenderAddress, tx.Value - 1); // we should have InsufficientFunds if balance < tx.Value + fee AcceptTxResult result = txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - txPool.GetPendingTransactions().Length.Should().Be(0); + txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.InsufficientFunds); EnsureSenderBalance(tx.SenderAddress, tx.Value); _blockTree.BlockAddedToMain += Raise.EventWith(_blockTree, new BlockReplacementEventArgs(Build.A.Block.WithGasLimit(10000000).TestObject)); result = txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); result.Should().Be(AcceptTxResult.InsufficientFunds); - txPool.GetPendingTransactions().Length.Should().Be(0); - } - - private static ISpecProvider GetLondonSpecProvider() - { - var specProvider = Substitute.For(); - specProvider.GetSpec(Arg.Any()).Returns(London.Instance); - specProvider.GetSpec(Arg.Any()).Returns(London.Instance); - return specProvider; - } - - private static ISpecProvider GetCancunSpecProvider() - { - var specProvider = Substitute.For(); - specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); - specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); - return specProvider; + txPool.GetPendingTransactionsCount().Should().Be(0); } [TestCase(false, false, ExpectedResult = nameof(AcceptTxResult.Accepted))] @@ -258,7 +241,7 @@ public void should_ignore_insufficient_funds_transactions() _txPool = CreatePool(); Transaction tx = Build.A.Transaction.SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.InsufficientFunds); } @@ -270,7 +253,7 @@ public void should_ignore_old_nonce_transactions() EnsureSenderBalance(tx); _stateProvider.IncrementNonce(tx.SenderAddress); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.OldNonce); } @@ -331,7 +314,7 @@ public void should_ignore_overflow_transactions() .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.Int256Overflow); } @@ -349,7 +332,7 @@ public void should_ignore_overflow_transactions_gas_premium_and_fee_cap() .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(tx.SenderAddress, UInt256.MaxValue); AcceptTxResult result = txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - txPool.GetPendingTransactions().Length.Should().Be(0); + txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.Int256Overflow); } @@ -363,7 +346,7 @@ public void should_ignore_block_gas_limit_exceeded() EnsureSenderBalance(tx); _headInfo.BlockGasLimit = Transaction.BaseTxGasCost * 4; AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.GasLimitExceeded); } @@ -384,7 +367,7 @@ public void should_accept_tx_when_base_fee_is_high() EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); result.Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); } [Test] @@ -396,7 +379,7 @@ public void should_ignore_tx_gas_limit_exceeded() .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.GasLimitExceeded); } @@ -443,7 +426,7 @@ public void should_handle_adding_tx_to_full_txPool_properly(int gasPrice, int va .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; tx.Value = (UInt256)(value * tx.GasLimit); EnsureSenderBalance(tx.SenderAddress, (UInt256)(15 * tx.GasLimit)); - _txPool.GetPendingTransactions().Length.Should().Be(30); + _txPool.GetPendingTransactionsCount().Should().Be(30); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.None); result.ToString().Should().Contain(expected); } @@ -486,9 +469,9 @@ public void should_handle_adding_1559_tx_to_full_txPool_properly(int gasPremium, .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; tx.Value = (UInt256)(value * tx.GasLimit); EnsureSenderBalance(tx.SenderAddress, (UInt256)(15 * tx.GasLimit)); - _txPool.GetPendingTransactions().Length.Should().Be(30); + _txPool.GetPendingTransactionsCount().Should().Be(30); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.None); - _txPool.GetPendingTransactions().Length.Should().Be(30); + _txPool.GetPendingTransactionsCount().Should().Be(30); result.ToString().Should().Contain(expected); } @@ -517,10 +500,10 @@ public void should_add_underpaid_txs_to_full_TxPool_only_if_local(bool isLocal) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA) .TestObject; EnsureSenderBalance(tx.SenderAddress, UInt256.MaxValue); - _txPool.GetPendingTransactions().Length.Should().Be(30); + _txPool.GetPendingTransactionsCount().Should().Be(30); _txPool.GetOwnPendingTransactions().Length.Should().Be(0); AcceptTxResult result = _txPool.SubmitTx(tx, txHandlingOptions); - _txPool.GetPendingTransactions().Length.Should().Be(30); + _txPool.GetPendingTransactionsCount().Should().Be(30); _txPool.GetOwnPendingTransactions().Length.Should().Be(isLocal ? 1 : 0); result.ToString().Should().Contain(isLocal ? nameof(AcceptTxResult.FeeTooLowToCompete) : nameof(AcceptTxResult.FeeTooLow)); } @@ -616,9 +599,9 @@ public void } } - _txPool.GetPendingTransactions().Length.Should().Be(8); + _txPool.GetPendingTransactionsCount().Should().Be(8); _txPool.SubmitTx(transactions[7], TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(9); + _txPool.GetPendingTransactionsCount().Should().Be(9); } [Test] @@ -647,7 +630,7 @@ public void should_discard_tx_because_of_overflow_of_cumulative_cost_of_this_tx_ } transactions[2].GasPrice = 5; - _txPool.GetPendingTransactions().Length.Should().Be(2); + _txPool.GetPendingTransactionsCount().Should().Be(2); _txPool.SubmitTx(transactions[2], TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Int256Overflow); } @@ -661,7 +644,8 @@ public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, t UInt256.MaxValue.Divide(GasCostOf.Transaction * 2, out UInt256 halfOfMaxGasPriceWithoutOverflow); Transaction firstTransaction = Build.A.Transaction - .WithType(TxType.EIP1559) + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerDataGas(UInt256.Zero) .WithNonce(UInt256.Zero) .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) @@ -669,15 +653,14 @@ public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, t _txPool.SubmitTx(firstTransaction, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); Transaction transactionWithPotentialOverflow = Build.A.Transaction - .WithMaxFeePerDataGas(supportsBlobs - ? UInt256.One - : UInt256.Zero) // simplification - it's still blob tx, but if MaxDataFee is 0, cost for blob - // data is 0, so we are able to test overflow well as only this field differs - .WithNonce(UInt256.One) - .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .WithShardBlobTxTypeAndFields() - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerDataGas(supportsBlobs + ? UInt256.One + : UInt256.Zero) + .WithNonce(UInt256.One) + .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(transactionWithPotentialOverflow, TxHandlingOptions.PersistentBroadcast).Should().Be(supportsBlobs ? AcceptTxResult.Int256Overflow : AcceptTxResult.Accepted); } @@ -802,7 +785,7 @@ public async Task should_remove_GasBottleneck_of_old_nonces() .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(transactions[i], TxHandlingOptions.PersistentBroadcast); } - _txPool.GetPendingTransactions().Length.Should().Be(5); + _txPool.GetPendingTransactionsCount().Should().Be(5); for (int i = 0; i < 3; i++) { @@ -895,7 +878,7 @@ public void broadcaster_should_work_well_when_there_are_no_txs_in_persistent_txs EnsureSenderBalance(transactionB); _txPool.SubmitTx(transactionB, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(2); + _txPool.GetPendingTransactionsCount().Should().Be(2); _txPool.GetOwnPendingTransactions().Length.Should().Be(1); Block block = Build.A.Block.WithTransactions(transactionA).TestObject; @@ -906,7 +889,7 @@ public void broadcaster_should_work_well_when_there_are_no_txs_in_persistent_txs _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockReplacementEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(200)); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); _txPool.GetOwnPendingTransactions().Length.Should().Be(1); } @@ -928,7 +911,7 @@ public async Task should_remove_transactions_concurrently() var secondTask = Task.Run(() => DeleteTransactionsFromPool(transactionsForSecondTask)); var thirdTask = Task.Run(() => DeleteTransactionsFromPool(transactionsForThirdTask)); await Task.WhenAll(firstTask, secondTask, thirdTask); - _txPool.GetPendingTransactions().Should().HaveCount(transactionsPerPeer * 7); + _txPool.GetPendingTransactionsCount().Should().Be(transactionsPerPeer * 7); } } @@ -964,7 +947,7 @@ public void should_add_pending_transactions(bool sameTransactionSenderPerPeer, b { _txPool = CreatePool(); AddTransactionsToPool(sameTransactionSenderPerPeer, sameNoncePerPeer); - _txPool.GetPendingTransactions().Length.Should().Be(expectedTransactions); + _txPool.GetPendingTransactionsCount().Should().Be(expectedTransactions); } [TestCase(true, true, 10)] @@ -976,7 +959,7 @@ public void should_remove_tx_from_txPool_when_included_in_block(bool sameTransac _txPool = CreatePool(); AddTransactionsToPool(sameTransactionSenderPerPeer, sameNoncePerPeer); - _txPool.GetPendingTransactions().Length.Should().Be(expectedTransactions); + _txPool.GetPendingTransactionsCount().Should().Be(expectedTransactions); Transaction[] transactions = _txPool.GetPendingTransactions(); Block block = Build.A.Block.WithTransactions(transactions).TestObject; @@ -987,7 +970,7 @@ public void should_remove_tx_from_txPool_when_included_in_block(bool sameTransac _blockTree.BlockAddedToMain += Raise.EventWith(new object(), blockReplacementEventArgs); manualResetEvent.WaitOne(TimeSpan.FromMilliseconds(200)); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); } [TestCase(true, true, 10)] @@ -999,7 +982,7 @@ public void should_not_remove_txHash_from_hashCache_when_tx_removed_because_of_i _txPool = CreatePool(); AddTransactionsToPool(sameTransactionSenderPerPeer, sameNoncePerPeer); - _txPool.GetPendingTransactions().Length.Should().Be(expectedTransactions); + _txPool.GetPendingTransactionsCount().Should().Be(expectedTransactions); Transaction[] transactions = _txPool.GetPendingTransactions(); Block block = Build.A.Block.WithTransactions(transactions).TestObject; @@ -1033,19 +1016,11 @@ public void should_return_feeTooLowTooCompete_result_when_trying_to_send_transac var result1 = _txPool.SubmitTx(GetTransaction(TestItem.PrivateKeyA, TestItem.AddressA), TxHandlingOptions.PersistentBroadcast | TxHandlingOptions.ManagedNonce); result1.Should().Be(AcceptTxResult.Accepted); _txPool.GetOwnPendingTransactions().Length.Should().Be(1); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); var result2 = _txPool.SubmitTx(GetTransaction(TestItem.PrivateKeyA, TestItem.AddressB), TxHandlingOptions.PersistentBroadcast | TxHandlingOptions.ManagedNonce); result2.Should().Be(AcceptTxResult.FeeTooLowToCompete); _txPool.GetOwnPendingTransactions().Length.Should().Be(1); - _txPool.GetPendingTransactions().Length.Should().Be(1); - } - - [Test] - public void Should_not_try_to_load_transactions_from_storage() - { - var transaction = Build.A.Transaction.SignedAndResolved().TestObject; - _txPool = CreatePool(); - _txPool.TryGetPendingTransaction(transaction.Hash, out var retrievedTransaction).Should().BeFalse(); + _txPool.GetPendingTransactionsCount().Should().Be(1); } [Test] @@ -1160,7 +1135,7 @@ public void should_accept_access_list_transactions_only_when_eip2930_enabled([Va .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(eip2930Enabled ? 1 : 0); + _txPool.GetPendingTransactionsCount().Should().Be(eip2930Enabled ? 1 : 0); result.Should().Be(eip2930Enabled ? AcceptTxResult.Accepted : AcceptTxResult.Invalid); } @@ -1175,7 +1150,7 @@ public void When_MaxFeePerGas_is_lower_than_MaxPriorityFeePerGas_tx_is_invalid() .TestObject; EnsureSenderBalance(tx); AcceptTxResult result = _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); result.Should().Be(AcceptTxResult.Invalid); } @@ -1191,7 +1166,7 @@ public void should_accept_zero_MaxFeePerGas_and_zero_MaxPriorityFee_1559_tx() .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); } [Test] @@ -1206,7 +1181,7 @@ public void should_reject_zero_MaxFeePerGas_and_positive_MaxPriorityFee_1559_tx( .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(0); + _txPool.GetPendingTransactionsCount().Should().Be(0); } [Test] @@ -1257,7 +1232,7 @@ public void should_replace_tx_with_same_sender_and_nonce_only_if_new_fee_is_at_l _txPool.SubmitTx(oldTx, TxHandlingOptions.PersistentBroadcast); _txPool.SubmitTx(newTx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); _txPool.GetPendingTransactions().First().Should().BeEquivalentTo(replaced ? newTx : oldTx); } @@ -1297,7 +1272,7 @@ public void should_replace_1559tx_with_same_sender_and_nonce_only_if_both_new_ma _txPool.SubmitTx(oldTx, TxHandlingOptions.PersistentBroadcast); _txPool.SubmitTx(newTx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); _txPool.GetPendingTransactions().First().Should().BeEquivalentTo(replaced ? newTx : oldTx); } @@ -1325,7 +1300,7 @@ public void should_always_replace_zero_fee_tx(int newGasPrice) _txPool.SubmitTx(oldTx, TxHandlingOptions.PersistentBroadcast); _txPool.SubmitTx(newTx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); _txPool.GetPendingTransactions().First().Should().BeEquivalentTo(newTx); } @@ -1355,7 +1330,7 @@ public void should_always_replace_zero_fee_tx_1559(int newMaxFeePerGas) _txPool.SubmitTx(oldTx, TxHandlingOptions.PersistentBroadcast); _txPool.SubmitTx(newTx, TxHandlingOptions.PersistentBroadcast); - _txPool.GetPendingTransactions().Length.Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(1); _txPool.GetPendingTransactions().First().Should().BeEquivalentTo(newTx); } @@ -1635,13 +1610,13 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g } [Test] - public void should_add_nonce_gap_tx_to_not_full_TxPool_only_if_is_not_blob_type() + public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool() { _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction - .WithType(TxType.EIP1559) + .WithShardBlobTxTypeAndFields() .WithMaxFeePerGas(UInt256.One) .WithMaxPriorityFeePerGas(UInt256.One) .WithNonce(UInt256.Zero) @@ -1654,6 +1629,23 @@ public void should_add_nonce_gap_tx_to_not_full_TxPool_only_if_is_not_blob_type( .WithNonce((UInt256)2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.NonceGap); + } + + [Test] + public void should_add_nonce_gap_tx_to_not_full_TxPool_if_is_not_blob_type() + { + _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + Transaction nonceGap1559Tx = Build.A.Transaction .WithType(TxType.EIP1559) .WithMaxFeePerGas(UInt256.One) @@ -1662,10 +1654,143 @@ public void should_add_nonce_gap_tx_to_not_full_TxPool_only_if_is_not_blob_type( .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.NonceGap); _txPool.SubmitTx(nonceGap1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); } + [Test] + public void should_not_allow_to_have_pending_transactions_of_both_blob_type_and_other([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) + { + Transaction GetBlobTx(UInt256 nonce) + { + return Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + Transaction Get1559Tx(UInt256 nonce) + { + return Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = firstIsBlob ? GetBlobTx(UInt256.Zero) : Get1559Tx(UInt256.Zero); + Transaction secondTx = secondIsBlob ? GetBlobTx(UInt256.One) : Get1559Tx(UInt256.One); + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted); + } + + [Test] + public async Task should_allow_to_have_pending_transaction_of_other_type_if_conflicting_one_was_included([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) + { + Transaction GetBlobTx(UInt256 nonce) + { + return Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + Transaction Get1559Tx(UInt256 nonce) + { + return Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = firstIsBlob ? GetBlobTx(UInt256.Zero) : Get1559Tx(UInt256.Zero); + Transaction secondTx = secondIsBlob ? GetBlobTx(UInt256.One) : Get1559Tx(UInt256.One); + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(firstIsBlob ? 0 : 1); + _txPool.GetPendingBlobTransactionsCount().Should().Be(firstIsBlob ? 1 : 0); + _stateProvider.IncrementNonce(TestItem.AddressA); + await RaiseBlockAddedToMainAndWaitForTransactions(1); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(0); + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + } + + [Test] + public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() + { + TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + + for (int i = 0; i < txPoolConfig.Size; i++) + { + Transaction tx = Build.A.Transaction + .WithNonce((UInt256)i) + .WithType(TxType.EIP1559) + .WithMaxFeePerGas((UInt256)(100 - i)) + .WithMaxPriorityFeePerGas((UInt256)(100 - i)) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); + } + + _txPool.GetPendingTransactionsCount().Should().Be(txPoolConfig.Size); + + Transaction feeTooLow1559Tx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; + + Transaction feeTooLowBlobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; + + + _txPool.SubmitTx(feeTooLow1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.FeeTooLow); + _txPool.SubmitTx(feeTooLowBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + } + + [Test] + public void should_add_blob_tx_and_return_when_requested() + { + TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction blobTxAdded = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(blobTxAdded, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.TryGetPendingTransaction(blobTxAdded.Hash!, out Transaction blobTxReturned); + + blobTxReturned.Should().BeEquivalentTo(blobTxAdded); + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] @@ -1730,6 +1855,22 @@ private ITxPoolPeer GetPeer(PublicKey publicKey) return peer; } + private static ISpecProvider GetLondonSpecProvider() + { + var specProvider = Substitute.For(); + specProvider.GetSpec(Arg.Any()).Returns(London.Instance); + specProvider.GetSpec(Arg.Any()).Returns(London.Instance); + return specProvider; + } + + private static ISpecProvider GetCancunSpecProvider() + { + var specProvider = Substitute.For(); + specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); + specProvider.GetSpec(Arg.Any()).Returns(Cancun.Instance); + return specProvider; + } + private Transaction[] AddTransactionsToPool(bool sameTransactionSenderPerPeer = true, bool sameNoncePerPeer = false, int transactionsPerPeer = 10) { var transactions = GetTransactions(GetPeers(transactionsPerPeer), sameTransactionSenderPerPeer, sameNoncePerPeer); diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index c55d49168da..e7258ef2695 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -76,6 +76,10 @@ namespace Nethermind.TxPool /// public static readonly AcceptTxResult SenderIsContract = new(12, nameof(SenderIsContract)); + /// + /// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs + /// + public static readonly AcceptTxResult PendingTxsOfOtherType = new(13, nameof(PendingTxsOfOtherType)); private int Id { get; } private string Code { get; } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs new file mode 100644 index 00000000000..072a3b17ff4 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -0,0 +1,85 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Caching; +using Nethermind.Core.Crypto; +using Nethermind.Logging; + +namespace Nethermind.TxPool.Collections; + +public class BlobTxDistinctSortedPool : TxDistinctSortedPool +{ + private readonly LruCache _blobTxCache = new(256, 256, "blob txs cache"); + + public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) + : base(capacity, comparer, logManager) + { + } + + protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); + + public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Transaction? removed) + { + Transaction lightBlobTx = new() + { + // Type = TxType.Blob, + Nonce = fullBlobTx.Nonce, + GasLimit = fullBlobTx.GasLimit, + GasPrice = fullBlobTx.GasPrice, // means MaxPriorityFeePerGas + DecodedMaxFeePerGas = fullBlobTx.DecodedMaxFeePerGas, + MaxFeePerDataGas = fullBlobTx.MaxFeePerDataGas, + Value = fullBlobTx.Value, + SenderAddress = fullBlobTx.SenderAddress, + Hash = hash.ToKeccak(), + }; + + if (base.TryInsert(hash, lightBlobTx, out removed)) + { + _blobTxCache.Set(hash, fullBlobTx); + // save to db fullBlobTx + return true; + } + + return false; + } + + public IEnumerable GetBlobTransactions() + { + // to refactor - it must return starting from the best + foreach (Transaction lightBlobTx in GetSnapshot()) + { + TryGetValue(lightBlobTx.Hash!, out Transaction? fullBlobTx); + yield return fullBlobTx!; + } + } + + public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) + { + if (base.TryGetValue(hash, out Transaction? lightBlobTx)) + { + // if (!_blobTxCache.TryGet(hash, out fullBlobTx)) + // { + // if (_blobTxsDb.TryGet(hash, fullBlobTx)) + // { + // _blobTxCache.Set(hash, fullBlobTx); + // return true; + // } + // } + // return false; + + return _blobTxCache.TryGet(hash, out fullBlobTx); + } + + fullBlobTx = default; + return false; + } + + protected override bool Remove(ValueKeccak hash, Transaction tx) + { + _blobTxCache.Delete(hash); + // delete from db here + return base.Remove(hash, tx); + } +} diff --git a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs index fd550cf972b..e6071d4947f 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/DistinctValueSortedPool.cs @@ -9,7 +9,7 @@ namespace Nethermind.TxPool.Collections { /// /// Keeps a distinct pool of with in groups based on . - /// Uses separate comparator to distinct between elements. If there is duplicate element added it uses ordering comparator and keeps the one that is larger. + /// Uses separate comparator to distinct between elements. If there is duplicate element added it uses ordering comparator and keeps the one that is larger. /// /// Type of keys of items, unique in pool. /// Type of items that are kept. diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 3b307b5b3ba..aefa890d5e1 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -173,7 +173,7 @@ protected void UpdateWorstValue() => /// Bucket for same sender transactions. /// If element was removed. False if element was not present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public bool TryRemove(TKey key, out TValue? value, [NotNullWhen(true)] out ICollection? bucket) => + private bool TryRemove(TKey key, out TValue? value, [NotNullWhen(true)] out ICollection? bucket) => TryRemove(key, false, out value, out bucket); private bool TryRemove(TKey key, bool evicted, [NotNullWhen(true)] out TValue? value, out ICollection? bucket) @@ -275,7 +275,7 @@ public bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) /// Element removed because of exceeding capacity /// If element was inserted. False if element was already present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public bool TryInsert(TKey key, TValue value, out TValue? removed) + public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) { if (CanInsert(key, value)) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/TxSortedPoolExtensions.cs b/src/Nethermind/Nethermind.TxPool/Collections/TxSortedPoolExtensions.cs index 93efff0b187..3a5c7111ad1 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/TxSortedPoolExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/TxSortedPoolExtensions.cs @@ -20,6 +20,9 @@ public static IComparer GetPoolUniqueTxComparerByNonce(this ICompar public static IComparer GetReplacementComparer(this IComparer comparer) => CompareReplacedTxByFee.Instance.ThenBy(comparer); + public static IComparer GetBlobReplacementComparer(this IComparer comparer) + => CompareReplacedBlobTx.Instance.ThenBy(comparer); + public static Address? MapTxToGroup(this Transaction value) => value.SenderAddress; } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs new file mode 100644 index 00000000000..234b9dcd00f --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -0,0 +1,53 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Int256; + +namespace Nethermind.TxPool.Comparison; + +public class CompareReplacedBlobTx : IComparer +{ + public static readonly CompareReplacedBlobTx Instance = new(); + + public CompareReplacedBlobTx() { } + + + //ToDo: it's copy-pasted std comparer, need to be adjusted for a needs of blobs + + + // To replace old transaction, new transaction needs to have fee higher by at least 10% (1/10) of current fee. + // It is required to avoid acceptance and propagation of transaction with almost the same fee as replaced one. + private const int PartOfFeeRequiredToIncrease = 10; + + public int Compare(Transaction? x, Transaction? y) + { + if (ReferenceEquals(x, y)) return 0; + if (ReferenceEquals(null, y)) return 1; + if (ReferenceEquals(null, x)) return -1; + + // always allow replacement of zero fee txs (in legacy txs MaxFeePerGas equals GasPrice) + if (y.MaxFeePerGas.IsZero) return -1; + + if (!x.Supports1559 && !y.Supports1559) + { + y.GasPrice.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpGasPrice); + int gasPriceResult = (y.GasPrice + bumpGasPrice).CompareTo(x.GasPrice); + // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease + // never return 0 - it's allowed or not + return gasPriceResult != 0 ? gasPriceResult : bumpGasPrice > 0 ? -1 : 1; + } + + /* MaxFeePerGas for legacy will be GasPrice and MaxPriorityFeePerGas will be GasPrice too + so we can compare legacy txs without any problems */ + y.MaxFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxFeePerGas); + if (y.MaxFeePerGas + bumpMaxFeePerGas > x.MaxFeePerGas) return 1; + + y.MaxPriorityFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxPriorityFeePerGas); + int result = (y.MaxPriorityFeePerGas + bumpMaxPriorityFeePerGas).CompareTo(x.MaxPriorityFeePerGas); + // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease + // never return 0 - it's allowed or not + return result != 0 ? result : (bumpMaxFeePerGas > 0 && bumpMaxPriorityFeePerGas > 0) ? -1 : 1; + } +} diff --git a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs index 1266c8c9af4..52dfe150ff5 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs @@ -14,11 +14,13 @@ namespace Nethermind.TxPool.Filters internal sealed class BalanceTooLowFilter : IIncomingTxFilter { private readonly TxDistinctSortedPool _txs; + private readonly TxDistinctSortedPool _blobTxs; private readonly ILogger _logger; - public BalanceTooLowFilter(TxDistinctSortedPool txs, ILogger logger) + public BalanceTooLowFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs, ILogger logger) { _txs = txs; + _blobTxs = blobTxs; _logger = logger; } @@ -34,11 +36,13 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO UInt256 cumulativeCost = UInt256.Zero; bool overflow = false; - Transaction[] transactions = _txs.GetBucketSnapshot(tx.SenderAddress!); // since unknownSenderFilter will run before this one + Transaction[] sameTypeTxs = tx.SupportsBlobs + ? _blobTxs.GetBucketSnapshot(tx.SenderAddress!) + : _txs.GetBucketSnapshot(tx.SenderAddress!); // since unknownSenderFilter will run before this one - for (int i = 0; i < transactions.Length; i++) + for (int i = 0; i < sameTypeTxs.Length; i++) { - Transaction otherTx = transactions[i]; + Transaction otherTx = sameTypeTxs[i]; if (otherTx.Nonce < account.Nonce) { continue; diff --git a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs index c7f0b4bbc9f..577f3324507 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs @@ -33,7 +33,7 @@ public FeeTooLowFilter(IChainHeadInfoProvider headInfo, TxDistinctSortedPool txs public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; - if (isLocal) + if (isLocal || tx.SupportsBlobs) { return AcceptTxResult.Accepted; } diff --git a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs index 283b386a455..e0566385bc4 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs @@ -15,11 +15,13 @@ namespace Nethermind.TxPool.Filters internal sealed class GapNonceFilter : IIncomingTxFilter { private readonly TxDistinctSortedPool _txs; + private readonly TxDistinctSortedPool _blobTxs; private readonly ILogger _logger; - public GapNonceFilter(TxDistinctSortedPool txs, ILogger logger) + public GapNonceFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs, ILogger logger) { _txs = txs; + _blobTxs = blobTxs; _logger = logger; } @@ -31,7 +33,9 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO return AcceptTxResult.Accepted; } - int numberOfSenderTxsInPending = _txs.GetBucketCount(tx.SenderAddress!); // since unknownSenderFilter will run before this one + int numberOfSenderTxsInPending = tx.SupportsBlobs + ? _blobTxs.GetBucketCount(tx.SenderAddress!) + : _txs.GetBucketCount(tx.SenderAddress!); // since unknownSenderFilter will run before this one UInt256 currentNonce = state.SenderAccount.Nonce; long nextNonceInOrder = (long)currentNonce + numberOfSenderTxsInPending; bool isTxNonceNextInOrder = tx.Nonce <= nextNonceInOrder; diff --git a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs new file mode 100644 index 00000000000..3eac786ff90 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs @@ -0,0 +1,34 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.TxPool.Collections; + +namespace Nethermind.TxPool.Filters; + +/// +/// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs +/// +public class TxTypeTxFilter : IIncomingTxFilter +{ + private readonly TxDistinctSortedPool _txs; + private readonly TxDistinctSortedPool _blobTxs; + + public TxTypeTxFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) + { + _txs = txs; + _blobTxs = blobTxs; + } + + public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) + { + if (tx.SupportsBlobs) + { + return _txs.TryGetBucket(tx.SenderAddress!, out _) ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted; + } + else + { + return _blobTxs.TryGetBucket(tx.SenderAddress!, out _) ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted; + } + } +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 8cd41cec842..68846987ca5 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -15,11 +15,17 @@ public interface ITxPool Transaction[] GetPendingTransactions(); /// - /// Grouped by sender address, sorted by nonce and later tx pool sorting + /// Non-blob txs grouped by sender address, sorted by nonce and later tx pool sorting /// /// IDictionary GetPendingTransactionsBySender(); + /// + /// Lazy return blob txs starting from the best one + /// + /// + IEnumerable GetBlobTransactions(); + /// /// from a specific sender, sorted by nonce and later tx pool sorting /// diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 546d095d04d..5cbe4bdf8a9 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -96,6 +96,14 @@ public static class Metrics [Description("Ratio of 1559-type transactions in the block.")] public static float Eip1559TransactionsRatio { get; set; } + [GaugeMetric] + [Description("Number of transactions in the block.")] + public static float BlobTransactionsInBlock { get; set; } + + [GaugeMetric] + [Description("Number of blobs in the block.")] + public static float BlobsInBlock { get; set; } + [GaugeMetric] [Description("Ratio of transactions in the block absent in hashCache.")] public static float DarkPoolRatioLevel1 { get; set; } @@ -107,5 +115,9 @@ public static class Metrics [GaugeMetric] [Description("Number of transactions in pool.")] public static float TransactionCount { get; set; } + + [GaugeMetric] + [Description("Number of blob transactions in pool.")] + public static float BlobTransactionCount { get; set; } } } diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index e1812faf9bb..df3337154fc 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -25,6 +25,8 @@ private NullTxPool() { } public IDictionary GetPendingTransactionsBySender() => new Dictionary(); + public IEnumerable GetBlobTransactions() => Array.Empty(); + public void AddPeer(ITxPoolPeer peer) { } public void RemovePeer(PublicKey nodeId) { } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index ffb988da550..dfe2a0761d5 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -98,8 +98,9 @@ public void Broadcast(Transaction tx, bool isPersistent) private void StartBroadcast(Transaction tx) { NotifyPeersAboutLocalTx(tx); - if (tx.Hash is not null) + if (tx.Hash is not null && !tx.SupportsBlobs) { + // somehow save only hashes of persistent blob txs? Or not add them here at all? Not add at all for now _persistentTxs.TryInsert(tx.Hash, tx); } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 94e255e6bb8..1f0764d62cf 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Channels; @@ -40,6 +41,8 @@ public class TxPool : ITxPool, IDisposable private readonly TxDistinctSortedPool _transactions; + private readonly BlobTxDistinctSortedPool _blobTransactions; + private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; @@ -95,6 +98,8 @@ public TxPool(IEthereumEcdsa ecdsa, AddNodeInfoEntryForTxPool(); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); + // we need some limit of blob txs, but extremely high to be limitless in practice + _blobTransactions = new BlobTxDistinctSortedPool(32 * MemoryAllowance.MemPoolSize, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _headInfo.HeadChanged += OnHeadChange; @@ -111,10 +116,11 @@ public TxPool(IEthereumEcdsa ecdsa, new NullHashTxFilter(), // needs to be first as it assigns the hash new AlreadyKnownTxFilter(_hashCache, _logger), new UnknownSenderFilter(ecdsa, _logger), + new TxTypeTxFilter(_transactions, _blobTransactions), // has to be after UnknownSenderFilter as it uses sender new BalanceZeroFilter(thereIsPriorityContract, _logger), - new BalanceTooLowFilter(_transactions, _logger), + new BalanceTooLowFilter(_transactions, _blobTransactions, _logger), new LowNonceFilter(_logger), // has to be after UnknownSenderFilter as it uses sender - new GapNonceFilter(_transactions, _logger), + new GapNonceFilter(_transactions, _blobTransactions, _logger), }; if (incomingTxFilter is not null) @@ -153,6 +159,15 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => internal Transaction[] GetOwnPendingTransactions() => _broadcaster.GetSnapshot(); + // public Transaction[] GetPendingBlobTransactions() => _blobTransactions.GetSnapshot(); + + public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; + + public IEnumerable GetBlobTransactions() => _blobTransactions.GetBlobTransactions(); + + // public Transaction[] GetPendingBlobTransactionsBySender(Address address) => + // _blobTransactions.GetBucketSnapshot(address); + private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { try @@ -185,6 +200,7 @@ private void ProcessNewHeads() UpdateBuckets(); _broadcaster.BroadcastPersistentTxs(); Metrics.TransactionCount = _transactions.Count; + Metrics.BlobTransactionCount = _blobTransactions.Count; } catch (Exception e) { @@ -225,6 +241,8 @@ private void RemoveProcessedTransactions(Transaction[] blockTransactions) long discoveredForPendingTxs = 0; long discoveredForHashCache = 0; long eip1559Txs = 0; + long blobTxs = 0; + long blobs = 0; for (int i = 0; i < blockTransactions.Length; i++) { @@ -245,6 +263,12 @@ private void RemoveProcessedTransactions(Transaction[] blockTransactions) { eip1559Txs++; } + + if (transaction.SupportsBlobs) + { + blobTxs++; + blobs += transaction.BlobVersionedHashes?.Length ?? 0; + } } long transactionsInBlock = blockTransactions.Length; @@ -253,6 +277,8 @@ private void RemoveProcessedTransactions(Transaction[] blockTransactions) Metrics.DarkPoolRatioLevel1 = (float)discoveredForHashCache / transactionsInBlock; Metrics.DarkPoolRatioLevel2 = (float)discoveredForPendingTxs / transactionsInBlock; Metrics.Eip1559TransactionsRatio = (float)eip1559Txs / transactionsInBlock; + Metrics.BlobTransactionsInBlock = blobTxs; + Metrics.BlobsInBlock = blobs; } } @@ -267,7 +293,8 @@ public void AddPeer(ITxPoolPeer peer) { if (_broadcaster.AddPeer(peer)) { - _broadcaster.BroadcastOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot()); + // worth to refactor and prepare tx snapshot in more efficient way + _broadcaster.BroadcastOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot().Concat(_blobTransactions.GetSnapshot()).ToArray()); if (_logger.IsTrace) _logger.Trace($"Added a peer to TX pool: {peer}"); } @@ -352,15 +379,17 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe bool eip1559Enabled = _specProvider.GetCurrentHeadSpec().IsEip1559Enabled; UInt256 effectiveGasPrice = tx.CalculateEffectiveGasPrice(eip1559Enabled, _headInfo.CurrentBaseFee); - _transactions.TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); + (tx.SupportsBlobs ? _blobTransactions : _transactions).TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); tx.GasBottleneck = (worstTx is null || effectiveGasPrice <= worstTx.GasBottleneck) ? effectiveGasPrice : worstTx.GasBottleneck; - bool inserted = _transactions.TryInsert(tx.Hash!, tx, out Transaction? removed); + bool inserted = (tx.SupportsBlobs + ? _blobTransactions.TryInsert(tx.Hash!, tx, out Transaction? removed) + : _transactions.TryInsert(tx.Hash!, tx, out removed)); if (inserted && tx.Hash != removed?.Hash) { - _transactions.UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); + (tx.SupportsBlobs ? _blobTransactions : _transactions).UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); Metrics.PendingTransactionsAdded++; if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } @@ -391,6 +420,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe _hashCache.SetLongTerm(tx.Hash!); NewPending?.Invoke(this, new TxEventArgs(tx)); Metrics.TransactionCount = _transactions.Count; + Metrics.BlobTransactionCount = _blobTransactions.Count; return AcceptTxResult.Accepted; } @@ -456,6 +486,8 @@ private void UpdateBuckets() if (_transactions.Count > _txPoolConfig.Size) if (_logger.IsWarn) _logger.Warn($"TxPool exceeds the config size {_transactions.Count}/{_txPoolConfig.Size}"); _transactions.UpdatePool(_accounts, _updateBucket); + + _blobTransactions.UpdatePool(_accounts, _updateBucket); } } @@ -523,6 +555,11 @@ public bool RemoveTransaction(Keccak? hash) lock (_locker) { hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction); + if (!hasBeenRemoved) + { + hasBeenRemoved = _blobTransactions.TryRemove(hash, out transaction); + } + if (transaction is null || !hasBeenRemoved) return false; if (hasBeenRemoved) @@ -542,28 +579,27 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) { lock (_locker) { - if (!_transactions.TryGetValue(hash, out transaction)) - { - _broadcaster.TryGetPersistentTx(hash, out transaction); - - // commented out as it puts too much pressure on the database - // and it not really required in any scenario - // * tx recovery usually will fetch from pending - // * get tx via RPC usually will fetch from block or from pending - // * internal tx pool scenarios are handled directly elsewhere - // transaction = _txStorage.Get(hash); - } + return _transactions.TryGetValue(hash, out transaction) + || _broadcaster.TryGetPersistentTx(hash, out transaction) + || _blobTransactions.TryGetValue(hash, out transaction); } - - return transaction is not null; } + // should own transactions (in broadcaster) be also checked here? + // maybe it should use NonceManager, as it already has info about local txs? public UInt256 GetLatestPendingNonce(Address address) { UInt256 maxPendingNonce = _accounts.GetAccount(address).Nonce; + bool hasPendingTxs = _transactions.GetBucketCount(address) > 0; + if (!hasPendingTxs && !(_blobTransactions.GetBucketCount(address) > 0)) + { + // sender doesn't have txs in any pool, quick return + return maxPendingNonce; + } + // we are not doing any updating, but lets just use a thread-safe method without any data copying like snapshot - _transactions.UpdateGroup(address, (_, transactions) => + (hasPendingTxs ? _transactions : _blobTransactions).UpdateGroup(address, (_, transactions) => { // This is under the assumption that the addressTransactions are sorted by Nonce. if (transactions.Count > 0) From 414fb8f39bb9ba6a8ade98680a757d12ea6c364f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 13 Jul 2023 21:46:39 +0200 Subject: [PATCH 005/148] fix metrics --- src/Nethermind/Nethermind.TxPool/Metrics.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 5cbe4bdf8a9..609da4987c9 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -98,11 +98,11 @@ public static class Metrics [GaugeMetric] [Description("Number of transactions in the block.")] - public static float BlobTransactionsInBlock { get; set; } + public static long BlobTransactionsInBlock { get; set; } [GaugeMetric] [Description("Number of blobs in the block.")] - public static float BlobsInBlock { get; set; } + public static long BlobsInBlock { get; set; } [GaugeMetric] [Description("Ratio of transactions in the block absent in hashCache.")] @@ -114,10 +114,10 @@ public static class Metrics [GaugeMetric] [Description("Number of transactions in pool.")] - public static float TransactionCount { get; set; } + public static long TransactionCount { get; set; } [GaugeMetric] [Description("Number of blob transactions in pool.")] - public static float BlobTransactionCount { get; set; } + public static long BlobTransactionCount { get; set; } } } From c508930443cbb0f356d7446ee28daaba56bb4e55 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 12:37:58 +0200 Subject: [PATCH 006/148] fix tx type, add test for replacing --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 33 +++++++++++++++++++ .../Collections/BlobTxDistinctSortedPool.cs | 2 +- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 7dc958e52de..f70b53c5dde 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1791,6 +1791,39 @@ public void should_add_blob_tx_and_return_when_requested() blobTxReturned.Should().BeEquivalentTo(blobTxAdded); } + [Test] + public void should_remove_replaced_blob_tx() + { + const int initialFee = 10; + TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction oldTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(initialFee) + .WithMaxPriorityFeePerGas(initialFee) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction newTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(initialFee * 2) + .WithMaxPriorityFeePerGas(initialFee * 2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(oldTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(oldTx.Hash!, out Transaction blobTxReturned).Should().BeTrue(); + blobTxReturned.Should().BeEquivalentTo(oldTx); + + _txPool.SubmitTx(newTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); + blobTxReturned.Should().BeEquivalentTo(newTx); + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 072a3b17ff4..c1f76ef5800 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -24,7 +24,7 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra { Transaction lightBlobTx = new() { - // Type = TxType.Blob, + Type = TxType.Blob, Nonce = fullBlobTx.Nonce, GasLimit = fullBlobTx.GasLimit, GasPrice = fullBlobTx.GasPrice, // means MaxPriorityFeePerGas From 45115fa5ff42c27ab6c550362f09a2da77112b8c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 13:02:19 +0200 Subject: [PATCH 007/148] adjust blob replacement comparer --- .../Comparison/CompareReplacedBlobTx.cs | 42 +++++-------------- 1 file changed, 11 insertions(+), 31 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index 234b9dcd00f..e5dd9dae123 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using Nethermind.Core; -using Nethermind.Int256; namespace Nethermind.TxPool.Comparison; @@ -13,41 +12,22 @@ public class CompareReplacedBlobTx : IComparer public CompareReplacedBlobTx() { } - - //ToDo: it's copy-pasted std comparer, need to be adjusted for a needs of blobs - - - // To replace old transaction, new transaction needs to have fee higher by at least 10% (1/10) of current fee. - // It is required to avoid acceptance and propagation of transaction with almost the same fee as replaced one. - private const int PartOfFeeRequiredToIncrease = 10; - + // To replace old blob transaction, new transaction needs to have fee at least 2x higher than current fee. + // 2x higher must be MaxPriorityFeePerGas, MaxFeePerGas and MaxFeePerDataGas public int Compare(Transaction? x, Transaction? y) { if (ReferenceEquals(x, y)) return 0; if (ReferenceEquals(null, y)) return 1; if (ReferenceEquals(null, x)) return -1; - // always allow replacement of zero fee txs (in legacy txs MaxFeePerGas equals GasPrice) - if (y.MaxFeePerGas.IsZero) return -1; - - if (!x.Supports1559 && !y.Supports1559) - { - y.GasPrice.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpGasPrice); - int gasPriceResult = (y.GasPrice + bumpGasPrice).CompareTo(x.GasPrice); - // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease - // never return 0 - it's allowed or not - return gasPriceResult != 0 ? gasPriceResult : bumpGasPrice > 0 ? -1 : 1; - } - - /* MaxFeePerGas for legacy will be GasPrice and MaxPriorityFeePerGas will be GasPrice too - so we can compare legacy txs without any problems */ - y.MaxFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxFeePerGas); - if (y.MaxFeePerGas + bumpMaxFeePerGas > x.MaxFeePerGas) return 1; - - y.MaxPriorityFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxPriorityFeePerGas); - int result = (y.MaxPriorityFeePerGas + bumpMaxPriorityFeePerGas).CompareTo(x.MaxPriorityFeePerGas); - // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease - // never return 0 - it's allowed or not - return result != 0 ? result : (bumpMaxFeePerGas > 0 && bumpMaxPriorityFeePerGas > 0) ? -1 : 1; + // always allow replacement of zero fee txs + if (y.MaxFeePerGas.IsZero) return -1; //ToDo: do we need it? + + if (y.MaxFeePerGas * 2 > x.MaxFeePerGas) return 1; + if (y.MaxPriorityFeePerGas * 2 > x.MaxPriorityFeePerGas) return 1; + if (y.MaxFeePerDataGas * 2 > x.MaxFeePerDataGas) return 1; + + // if we are here, it means that all new fees are at least 2x higher than old ones, so replacement is allowed + return -1; } } From c0db165923ec61a6fb5d5fd0c090f3d00cd4e9fb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 13:42:44 +0200 Subject: [PATCH 008/148] add blob tx db and metrics --- src/Nethermind/Nethermind.Db/DbNames.cs | 1 + src/Nethermind/Nethermind.Db/IDbProvider.cs | 2 ++ src/Nethermind/Nethermind.Db/Metrics.cs | 8 ++++++++ src/Nethermind/Nethermind.Db/StandardDbInitializer.cs | 1 + 4 files changed, 12 insertions(+) diff --git a/src/Nethermind/Nethermind.Db/DbNames.cs b/src/Nethermind/Nethermind.Db/DbNames.cs index a0f4652b6eb..6a234c67df1 100644 --- a/src/Nethermind/Nethermind.Db/DbNames.cs +++ b/src/Nethermind/Nethermind.Db/DbNames.cs @@ -16,5 +16,6 @@ public static class DbNames public const string Witness = "witness"; public const string CHT = "canonicalHashTrie"; public const string Metadata = "metadata"; + public const string BlobTransactions = "blobTransactions"; } } diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index 5625941f1f5..d6a3855a886 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -32,6 +32,8 @@ public interface IDbProvider : IDisposable public IDb MetadataDb => GetDb(DbNames.Metadata); + public IDb BlobTransactionsDb => GetDb(DbNames.BlobTransactions); + T GetDb(string dbName) where T : class, IDb; void RegisterDb(string dbName, T db) where T : class, IDb; diff --git a/src/Nethermind/Nethermind.Db/Metrics.cs b/src/Nethermind/Nethermind.Db/Metrics.cs index 9952dd6f773..62fb429df0b 100644 --- a/src/Nethermind/Nethermind.Db/Metrics.cs +++ b/src/Nethermind/Nethermind.Db/Metrics.cs @@ -118,6 +118,14 @@ public static class Metrics [Description("Number of Metadata DB writes.")] public static long MetadataDbWrites { get; set; } + [CounterMetric] + [Description("Number of BlobTransactions DB reads.")] + public static long BlobTransactionsDbReads { get; set; } + + [CounterMetric] + [Description("Number of BlobTransactions DB writes.")] + public static long BlobTransactionsDbWrites { get; set; } + [GaugeMetric] [Description("Indicator if StadeDb is being pruned.")] public static int StateDbPruning { get; set; } diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index 8f5f0d7f199..b6ff4e65055 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -65,6 +65,7 @@ private void RegisterAll(bool useReceiptsDb) RegisterCustomDb(DbNames.Receipts, () => new ReadOnlyColumnsDb(new MemColumnsDb(), false)); } RegisterDb(BuildRocksDbSettings(DbNames.Metadata, () => Metrics.MetadataDbReads++, () => Metrics.MetadataDbWrites++)); + RegisterDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); } private RocksDbSettings BuildRocksDbSettings(string dbName, Action updateReadsMetrics, Action updateWriteMetrics, bool deleteOnStart = false) From 15087f4dc0005ae1d6f2e1071bea137b8292785e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 14:04:46 +0200 Subject: [PATCH 009/148] fix test --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 7 ++++--- .../Collections/BlobTxDistinctSortedPool.cs | 1 + 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index f70b53c5dde..d4e7667fcf4 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1795,7 +1795,7 @@ public void should_add_blob_tx_and_return_when_requested() public void should_remove_replaced_blob_tx() { const int initialFee = 10; - TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1809,8 +1809,9 @@ public void should_remove_replaced_blob_tx() Transaction newTx = Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(initialFee * 2) - .WithMaxPriorityFeePerGas(initialFee * 2) + .WithMaxFeePerGas(oldTx.MaxFeePerGas * 2) + .WithMaxPriorityFeePerGas(oldTx.MaxPriorityFeePerGas * 2) + .WithMaxFeePerDataGas(oldTx.MaxFeePerDataGas * 2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(oldTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index c1f76ef5800..4303fe8ba2d 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -30,6 +30,7 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra GasPrice = fullBlobTx.GasPrice, // means MaxPriorityFeePerGas DecodedMaxFeePerGas = fullBlobTx.DecodedMaxFeePerGas, MaxFeePerDataGas = fullBlobTx.MaxFeePerDataGas, + BlobVersionedHashes = new byte[fullBlobTx.BlobVersionedHashes!.Length][], Value = fullBlobTx.Value, SenderAddress = fullBlobTx.SenderAddress, Hash = hash.ToKeccak(), From c6ea9fef3494b360440f12347d0e079d99497860 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 14:14:21 +0200 Subject: [PATCH 010/148] resurrect old TxStorage class and adjust a bit for blobs --- .../Nethermind.TxPool/BlobTxStorage.cs | 54 +++++++++++++++++++ .../Nethermind.TxPool/ITxStorage.cs | 15 ++++++ 2 files changed, 69 insertions(+) create mode 100644 src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs create mode 100644 src/Nethermind/Nethermind.TxPool/ITxStorage.cs diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs new file mode 100644 index 00000000000..aa3e3eff074 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -0,0 +1,54 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Linq; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Db; +using Nethermind.Serialization.Rlp; + +namespace Nethermind.TxPool; + +public class BlobTxStorage : ITxStorage +{ + private readonly IDb _database; + + public BlobTxStorage(IDb database) + { + _database = database ?? throw new ArgumentNullException(nameof(database)); + } + + public Transaction? Get(Keccak hash) => Decode(_database.Get(hash)); + + public Transaction?[] GetAll() + { + byte[][] transactionsBytes = _database.GetAllValues().ToArray(); + if (transactionsBytes.Length == 0) + { + return Array.Empty(); + } + + Transaction?[] transactions = new Transaction[transactionsBytes.Length]; + for (int i = 0; i < transactionsBytes.Length; i++) + { + transactions[i] = Decode(transactionsBytes[i]); + } + + return transactions; + } + + private static Transaction? Decode(byte[]? bytes) => bytes == null ? null : Rlp.Decode(new Rlp(bytes)); + + public void Add(Transaction transaction) + { + if (transaction == null || transaction.Hash == null) + { + throw new ArgumentNullException(nameof(transaction)); + } + + _database.Set(transaction.Hash, Rlp.Encode(transaction, RlpBehaviors.None).Bytes); + } + + public void Remove(Keccak hash) => _database.Remove(hash.Bytes); +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs new file mode 100644 index 00000000000..4a057d39010 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -0,0 +1,15 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.TxPool; + +public interface ITxStorage +{ + Transaction? Get(Keccak hash); + Transaction?[] GetAll(); + void Add(Transaction transaction); + void Remove(Keccak hash); +} From d1c66c10d95231b4130ead46477fa5dc4048693a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 14:26:46 +0200 Subject: [PATCH 011/148] pass blob storage to tx pool, adjust tests --- .../Nethermind.Blockchain.Test/ReorgTests.cs | 1 + .../CliqueBlockProducerTests.cs | 8 +++++- .../InitializeBlockchainAuRa.cs | 1 + .../Blockchain/TestBlockchain.cs | 1 + .../Steps/InitializeBlockchain.cs | 1 + .../Modules/ParityRpcModuleTests.cs | 27 ++++++++++++++----- .../SyncThreadTests.cs | 9 +++++-- .../Nethermind.TxPool.Test/TxPoolTests.cs | 1 + src/Nethermind/Nethermind.TxPool/TxPool.cs | 9 ++++--- 9 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index 8795add9d9c..d1627fd9ced 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -61,6 +61,7 @@ public void Setup() LimboLogs.Instance); TxPool.TxPool txPool = new( ecdsa, + new BlobTxStorage(new MemDb()), new ChainHeadInfoProvider(specProvider, _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs index b905d8e6a49..643cb9b670b 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs @@ -107,7 +107,13 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(_ethereumEcdsa, new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(GoerliSpecProvider.Instance), blockTree, stateProvider), new TxPoolConfig(), new TxValidator(goerliSpecProvider.ChainId), _logManager, transactionComparerProvider.GetDefaultComparer()); + TxPool.TxPool txPool = new(_ethereumEcdsa, + new BlobTxStorage(new MemDb()), + new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(GoerliSpecProvider.Instance), blockTree, stateProvider), + new TxPoolConfig(), + new TxValidator(goerliSpecProvider.ChainId), + _logManager, + transactionComparerProvider.GetDefaultComparer()); _pools[privateKey] = txPool; BlockhashProvider blockhashProvider = new(blockTree, LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs index 57cca506565..4fcb7b71b63 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs @@ -273,6 +273,7 @@ protected override TxPool.TxPool CreateTxPool() return new TxPool.TxPool( _api.EthereumEcdsa, + new BlobTxStorage(_api.DbProvider.BlobTransactionsDb), new ChainHeadInfoProvider(_api.SpecProvider, _api.BlockTree, _api.StateReader), NethermindApi.Config(), _api.TxValidator, diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index 59ab6390e1e..44cde083420 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -265,6 +265,7 @@ protected virtual IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolTx protected virtual TxPool.TxPool CreateTxPool() => new( EthereumEcdsa, + new BlobTxStorage(new MemDb()), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), new TxPoolConfig(), new TxValidator(SpecProvider.ChainId), diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index 943d1358b20..00d6f916f72 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -329,6 +329,7 @@ protected virtual IHealthHintService CreateHealthHintService() => protected virtual TxPool.TxPool CreateTxPool() => new(_api.EthereumEcdsa!, + new BlobTxStorage(_api.DbProvider!.BlobTransactionsDb), new ChainHeadInfoProvider(_api.SpecProvider!, _api.BlockTree!, _api.StateReader!), _api.Config(), _api.TxValidator!, diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index 1691cdc8cc4..28ce4008ac7 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -72,8 +72,13 @@ public void Initialize() ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, stateProvider), new TxPoolConfig(), - new TxValidator(specProvider.ChainId), LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); + TxPool.TxPool txPool = new(ethereumEcdsa, + new BlobTxStorage(new MemDb()), + new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, stateProvider), + new TxPoolConfig(), + new TxValidator(specProvider.ChainId), + LimboLogs.Instance, + transactionComparerProvider.GetDefaultComparer()); IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); @@ -345,8 +350,13 @@ public void parity_netPeers_empty_ActivePeers() ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, new WorldState(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), new TxPoolConfig(), - new TxValidator(specProvider.ChainId), LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); + TxPool.TxPool txPool = new(ethereumEcdsa, + new BlobTxStorage(new MemDb()), + new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, new WorldState(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), + new TxPoolConfig(), + new TxValidator(specProvider.ChainId), + LimboLogs.Instance, + transactionComparerProvider.GetDefaultComparer()); IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); @@ -377,8 +387,13 @@ public void parity_netPeers_null_ActivePeers() IBlockTree blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, LimboLogs.Instance); ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, new ChainHeadInfoProvider(specProvider, blockTree, new StateReader(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), new TxPoolConfig(), - new TxValidator(specProvider.ChainId), LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); + TxPool.TxPool txPool = new(ethereumEcdsa, + new BlobTxStorage(new MemDb()), + new ChainHeadInfoProvider(specProvider, blockTree, new StateReader(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), + new TxPoolConfig(), + new TxValidator(specProvider.ChainId), + LimboLogs.Instance, + transactionComparerProvider.GetDefaultComparer()); IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index a3ca85cfec9..a28752c6ff6 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -271,8 +271,13 @@ private SyncTestContext CreateSyncManager(int index) ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, tree); - TxPool.TxPool txPool = new(ecdsa, new ChainHeadInfoProvider(specProvider, tree, stateReader), - new TxPoolConfig(), new TxValidator(specProvider.ChainId), logManager, transactionComparerProvider.GetDefaultComparer()); + TxPool.TxPool txPool = new(ecdsa, + new BlobTxStorage(new MemDb()), + new ChainHeadInfoProvider(specProvider, tree, stateReader), + new TxPoolConfig(), + new TxValidator(specProvider.ChainId), + logManager, + transactionComparerProvider.GetDefaultComparer()); BlockhashProvider blockhashProvider = new(tree, LimboLogs.Instance); VirtualMachine virtualMachine = new(blockhashProvider, specProvider, logManager); diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index d4e7667fcf4..a56ff718b84 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1871,6 +1871,7 @@ private TxPool CreatePool( return new TxPool( _ethereumEcdsa, + new BlobTxStorage(new MemDb()), _headInfo, config ?? new TxPoolConfig() { GasLimit = _txGasLimit }, new TxValidator(_specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 1f0764d62cf..50049c52c50 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -36,17 +36,15 @@ public class TxPool : ITxPool, IDisposable private readonly IIncomingTxFilter[] _postHashFilters; private readonly HashCache _hashCache = new(); - private readonly TxBroadcaster _broadcaster; private readonly TxDistinctSortedPool _transactions; - private readonly BlobTxDistinctSortedPool _blobTransactions; - private readonly IChainHeadSpecProvider _specProvider; + private readonly ITxStorage _blobTxStorage; + private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; - private readonly IChainHeadInfoProvider _headInfo; private readonly ITxPoolConfig _txPoolConfig; @@ -69,6 +67,7 @@ public class TxPool : ITxPool, IDisposable /// (by miners or validators) or simply informing other nodes about known pending transactions (broadcasting). /// /// Used to recover sender addresses from transaction signatures. + /// /// /// /// @@ -79,6 +78,7 @@ public class TxPool : ITxPool, IDisposable /// /// Tx storage used to reject known transactions. public TxPool(IEthereumEcdsa ecdsa, + ITxStorage blobTxStorage, IChainHeadInfoProvider chainHeadInfoProvider, ITxPoolConfig txPoolConfig, ITxValidator validator, @@ -89,6 +89,7 @@ public TxPool(IEthereumEcdsa ecdsa, bool thereIsPriorityContract = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); + _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; _accounts = _headInfo.AccountStateProvider; From a6a65c52c3fc06076f4bb0e3a1e6f6e84db8dfe5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 19:32:37 +0200 Subject: [PATCH 012/148] handle db writes/reads and add test --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 19 ++++++++- .../Nethermind.TxPool/BlobTxStorage.cs | 39 ++++++++++++------- .../Collections/BlobTxDistinctSortedPool.cs | 35 ++++++++++------- .../Collections/SortedPool.cs | 2 +- .../Nethermind.TxPool/ITxStorage.cs | 7 ++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 +- 6 files changed, 69 insertions(+), 37 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index a56ff718b84..f6b9c3c054d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1796,7 +1796,8 @@ public void should_remove_replaced_blob_tx() { const int initialFee = 10; TxPoolConfig txPoolConfig = new() { Size = 10 }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + BlobTxStorage blobTxStorage = new(new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction oldTx = Build.A.Transaction @@ -1814,15 +1815,27 @@ public void should_remove_replaced_blob_tx() .WithMaxFeePerDataGas(oldTx.MaxFeePerDataGas * 2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(oldTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.GetPendingBlobTransactionsCount().Should().Be(1); _txPool.TryGetPendingTransaction(oldTx.Hash!, out Transaction blobTxReturned).Should().BeTrue(); blobTxReturned.Should().BeEquivalentTo(oldTx); + blobTxStorage.TryGet(oldTx.Hash, out Transaction blobTxFromDb).Should().BeTrue(); + blobTxFromDb.Should().BeEquivalentTo(oldTx, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex _txPool.SubmitTx(newTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.GetPendingBlobTransactionsCount().Should().Be(1); _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); blobTxReturned.Should().BeEquivalentTo(newTx); + blobTxStorage.TryGet(oldTx.Hash, out blobTxFromDb).Should().BeFalse(); + blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); + blobTxFromDb.Should().BeEquivalentTo(newTx, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex } [TestCase(0, 97)] @@ -1860,18 +1873,20 @@ private TxPool CreatePool( ISpecProvider specProvider = null, ChainHeadInfoProvider chainHeadInfoProvider = null, IIncomingTxFilter incomingTxFilter = null, + ITxStorage txStorage = null, bool thereIsPriorityContract = false) { specProvider ??= MainnetSpecProvider.Instance; ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, _blockTree); + txStorage ??= new BlobTxStorage(new MemDb()); _headInfo = chainHeadInfoProvider; _headInfo ??= new ChainHeadInfoProvider(specProvider, _blockTree, _stateProvider); return new TxPool( _ethereumEcdsa, - new BlobTxStorage(new MemDb()), + txStorage, _headInfo, config ?? new TxPoolConfig() { GasLimit = _txGasLimit }, new TxValidator(_specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index aa3e3eff074..d9d64135cc5 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -2,7 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; -using System.Linq; +using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Db; @@ -19,27 +19,38 @@ public BlobTxStorage(IDb database) _database = database ?? throw new ArgumentNullException(nameof(database)); } - public Transaction? Get(Keccak hash) => Decode(_database.Get(hash)); + public bool TryGet(Keccak hash, out Transaction? transaction) => TryDecode(_database.Get(hash), out transaction); - public Transaction?[] GetAll() + public IEnumerable GetAll() { - byte[][] transactionsBytes = _database.GetAllValues().ToArray(); - if (transactionsBytes.Length == 0) + foreach (byte[] txBytes in _database.GetAllValues()) { - return Array.Empty(); + if (TryDecode(txBytes, out Transaction? transaction)) + { + yield return transaction!; + } } + } - Transaction?[] transactions = new Transaction[transactionsBytes.Length]; - for (int i = 0; i < transactionsBytes.Length; i++) + private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) + { + if (txBytes is not null) { - transactions[i] = Decode(transactionsBytes[i]); + try + { + transaction = Rlp.Decode(new Rlp(txBytes), RlpBehaviors.InMempoolForm); + return true; + } + catch (Exception) + { + // ignored + } } - return transactions; + transaction = default; + return false; } - private static Transaction? Decode(byte[]? bytes) => bytes == null ? null : Rlp.Decode(new Rlp(bytes)); - public void Add(Transaction transaction) { if (transaction == null || transaction.Hash == null) @@ -47,8 +58,8 @@ public void Add(Transaction transaction) throw new ArgumentNullException(nameof(transaction)); } - _database.Set(transaction.Hash, Rlp.Encode(transaction, RlpBehaviors.None).Bytes); + _database.Set(transaction.Hash, Rlp.Encode(transaction, RlpBehaviors.InMempoolForm).Bytes); } - public void Remove(Keccak hash) => _database.Remove(hash.Bytes); + public void Delete(ValueKeccak hash) => _database.Remove(hash.Bytes); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 4303fe8ba2d..a876b05bb8c 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -11,16 +11,21 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { + private readonly ITxStorage _blobTxStorage; private readonly LruCache _blobTxCache = new(256, 256, "blob txs cache"); - public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) + public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, logManager) { + _blobTxStorage = blobTxStorage; } protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Transaction? removed) + public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) => + TryInsert(fullBlobTx.Hash!, fullBlobTx, out removed); + + private bool TryInsert(Keccak hash, Transaction fullBlobTx, out Transaction? removed) { Transaction lightBlobTx = new() { @@ -33,13 +38,14 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra BlobVersionedHashes = new byte[fullBlobTx.BlobVersionedHashes!.Length][], Value = fullBlobTx.Value, SenderAddress = fullBlobTx.SenderAddress, - Hash = hash.ToKeccak(), + Hash = hash, + GasBottleneck = fullBlobTx.GasBottleneck, }; if (base.TryInsert(hash, lightBlobTx, out removed)) { _blobTxCache.Set(hash, fullBlobTx); - // save to db fullBlobTx + _blobTxStorage.Add(fullBlobTx); return true; } @@ -60,17 +66,16 @@ public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) { if (base.TryGetValue(hash, out Transaction? lightBlobTx)) { - // if (!_blobTxCache.TryGet(hash, out fullBlobTx)) - // { - // if (_blobTxsDb.TryGet(hash, fullBlobTx)) - // { - // _blobTxCache.Set(hash, fullBlobTx); - // return true; - // } - // } - // return false; + if (_blobTxCache.TryGet(hash, out fullBlobTx)) + { + return true; + } - return _blobTxCache.TryGet(hash, out fullBlobTx); + if (_blobTxStorage.TryGet(hash, out fullBlobTx)) + { + _blobTxCache.Set(hash, fullBlobTx!); + return true; + } } fullBlobTx = default; @@ -80,7 +85,7 @@ public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); - // delete from db here + _blobTxStorage.Delete(hash); return base.Remove(hash, tx); } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index aefa890d5e1..76988734ddb 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -275,7 +275,7 @@ public bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) /// Element removed because of exceeding capacity /// If element was inserted. False if element was already present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) + public bool TryInsert(TKey key, TValue value, out TValue? removed) { if (CanInsert(key, value)) { diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 4a057d39010..1fb6b206b1f 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -8,8 +9,8 @@ namespace Nethermind.TxPool; public interface ITxStorage { - Transaction? Get(Keccak hash); - Transaction?[] GetAll(); + bool TryGet(Keccak hash, out Transaction? transaction); + IEnumerable GetAll(); void Add(Transaction transaction); - void Remove(Keccak hash); + void Delete(ValueKeccak hash); } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 50049c52c50..e26d667a0e5 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -100,7 +100,7 @@ public TxPool(IEthereumEcdsa ecdsa, _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); // we need some limit of blob txs, but extremely high to be limitless in practice - _blobTransactions = new BlobTxDistinctSortedPool(32 * MemoryAllowance.MemPoolSize, comparer, logManager); + _blobTransactions = new BlobTxDistinctSortedPool(_blobTxStorage, 32 * MemoryAllowance.MemPoolSize, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _headInfo.HeadChanged += OnHeadChange; @@ -386,7 +386,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe : worstTx.GasBottleneck; bool inserted = (tx.SupportsBlobs - ? _blobTransactions.TryInsert(tx.Hash!, tx, out Transaction? removed) + ? _blobTransactions.TryInsert(tx, out Transaction? removed) : _transactions.TryInsert(tx.Hash!, tx, out removed)); if (inserted && tx.Hash != removed?.Hash) { From 5cf90f5cefaf40641fd9ed73014139aa90f3ac75 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 20:07:52 +0200 Subject: [PATCH 013/148] recreate light collection after restart --- .../Collections/BlobTxDistinctSortedPool.cs | 52 +++++++++++++------ 1 file changed, 35 insertions(+), 17 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index a876b05bb8c..1ac9c139d50 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -13,48 +13,66 @@ public class BlobTxDistinctSortedPool : TxDistinctSortedPool { private readonly ITxStorage _blobTxStorage; private readonly LruCache _blobTxCache = new(256, 256, "blob txs cache"); + private readonly ILogger _logger; public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, logManager) { _blobTxStorage = blobTxStorage; + _logger = logManager.GetClassLogger(); + + RecreateLightTxCollectionAndCache(blobTxStorage); } protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) => - TryInsert(fullBlobTx.Hash!, fullBlobTx, out removed); + private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) + { + if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); + foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) + { + if (base.TryInsert(fullBlobTx.Hash, CreateLightTx(fullBlobTx))) + { + _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); + } + } + } + + public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) + { + Transaction lightBlobTx = CreateLightTx(fullBlobTx); + + if (base.TryInsert(fullBlobTx.Hash, lightBlobTx, out removed)) + { + _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); + _blobTxStorage.Add(fullBlobTx); + return true; + } + + return false; + } - private bool TryInsert(Keccak hash, Transaction fullBlobTx, out Transaction? removed) + private Transaction CreateLightTx(Transaction fullBlobTx) { - Transaction lightBlobTx = new() + return new Transaction { Type = TxType.Blob, + Hash = fullBlobTx.Hash, + SenderAddress = fullBlobTx.SenderAddress, Nonce = fullBlobTx.Nonce, + Value = fullBlobTx.Value, GasLimit = fullBlobTx.GasLimit, GasPrice = fullBlobTx.GasPrice, // means MaxPriorityFeePerGas DecodedMaxFeePerGas = fullBlobTx.DecodedMaxFeePerGas, MaxFeePerDataGas = fullBlobTx.MaxFeePerDataGas, BlobVersionedHashes = new byte[fullBlobTx.BlobVersionedHashes!.Length][], - Value = fullBlobTx.Value, - SenderAddress = fullBlobTx.SenderAddress, - Hash = hash, GasBottleneck = fullBlobTx.GasBottleneck, }; - - if (base.TryInsert(hash, lightBlobTx, out removed)) - { - _blobTxCache.Set(hash, fullBlobTx); - _blobTxStorage.Add(fullBlobTx); - return true; - } - - return false; } public IEnumerable GetBlobTransactions() { - // to refactor - it must return starting from the best + // to refactor - it must enumerable starting from the best foreach (Transaction lightBlobTx in GetSnapshot()) { TryGetValue(lightBlobTx.Hash!, out Transaction? fullBlobTx); From 2e91dcb505bf3233877ef2aaae331163d4b1e16f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 20:33:06 +0200 Subject: [PATCH 014/148] adjust broadcaster test --- .../TxBroadcasterTests.cs | 145 ++++++++---------- 1 file changed, 68 insertions(+), 77 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index ec55888799d..7904c5d3c71 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -153,83 +153,74 @@ public void should_pick_best_persistent_txs_to_broadcast(int threshold) expectedTxs.Should().BeEquivalentTo(pickedTxs); } - // commented out as for now we are not adding blob txs to persistent at all. - // work is still in progress so it might be back - keeping commented out for now - // [TestCase(1)] - // [TestCase(2)] - // [TestCase(25)] - // [TestCase(50)] - // [TestCase(99)] - // [TestCase(100)] - // [TestCase(101)] - // [TestCase(1000)] - // public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast(int threshold) - // { - // _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; - // _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); - // _headInfo.CurrentBaseFee.Returns(0.GWei()); - // - // int addedTxsCount = TestItem.PrivateKeys.Length; - // Transaction[] transactions = new Transaction[addedTxsCount]; - // - // for (int i = 0; i < addedTxsCount; i++) - // { - // bool isBlob = i % 10 == 0; - // transactions[i] = Build.A.Transaction - // .WithGasPrice((addedTxsCount - i - 1).GWei()) - // .WithType(isBlob ? TxType.Blob : TxType.Legacy) //some part of txs (10%) is blob type - // .WithShardBlobTxTypeAndFieldsIfBlobTx() - // .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) - // .TestObject; - // - // _broadcaster.Broadcast(transactions[i], true); - // } - // - // _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); - // - // (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); - // - // int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); - // int expectedCountOfBlobHashes = expectedCountTotal / 10 + 1; - // int expectedCountOfNonBlobTxs = expectedCountTotal - expectedCountOfBlobHashes; - // if (expectedCountOfNonBlobTxs > 0) - // { - // pickedTxs.Count.Should().Be(expectedCountOfNonBlobTxs); - // } - // else - // { - // pickedTxs.Should().BeNull(); - // } - // - // if (expectedCountOfBlobHashes > 0) - // { - // pickedHashes.Count.Should().Be(expectedCountOfBlobHashes); - // } - // else - // { - // pickedHashes.Should().BeNull(); - // } - // - // List expectedTxs = new(); - // List expectedHashes = new(); - // - // for (int i = 0; i < expectedCountTotal; i++) - // { - // Transaction tx = transactions[i]; - // - // if (!tx.SupportsBlobs) - // { - // expectedTxs.Add(tx); - // } - // else - // { - // expectedHashes.Add(tx); - // } - // } - // - // expectedTxs.Should().BeEquivalentTo(pickedTxs); - // expectedHashes.Should().BeEquivalentTo(pickedHashes); - // } + [Test] + public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast([Values(1, 2, 25, 50, 99, 100, 101, 1000)] int threshold) + { + _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; + _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + _headInfo.CurrentBaseFee.Returns(0.GWei()); + + int addedTxsCount = TestItem.PrivateKeys.Length; + Transaction[] transactions = new Transaction[addedTxsCount]; + + for (int i = 0; i < addedTxsCount; i++) + { + bool isLarge = i % 10 == 0; + transactions[i] = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithGasPrice((addedTxsCount - i - 1).GWei()) + .WithData(isLarge ? new byte[4 * 1024] : Array.Empty()) //some part of txs (10%) is large (>4KB) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) + .TestObject; + + _broadcaster.Broadcast(transactions[i], true); + } + + _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); + + (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); + + int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); + int expectedCountOfHashes = expectedCountTotal / 10 + 1; + int expectedCountOfFullTxs = expectedCountTotal - expectedCountOfHashes; + if (expectedCountOfFullTxs > 0) + { + pickedTxs.Count.Should().Be(expectedCountOfFullTxs); + } + else + { + pickedTxs.Should().BeNull(); + } + + if (expectedCountOfHashes > 0) + { + pickedHashes.Count.Should().Be(expectedCountOfHashes); + } + else + { + pickedHashes.Should().BeNull(); + } + + List expectedTxs = new(); + List expectedHashes = new(); + + for (int i = 0; i < expectedCountTotal; i++) + { + Transaction tx = transactions[i]; + + if (tx.CanBeBroadcast()) + { + expectedTxs.Add(tx); + } + else + { + expectedHashes.Add(tx); + } + } + + expectedTxs.Should().BeEquivalentTo(pickedTxs); + expectedHashes.Should().BeEquivalentTo(pickedHashes); + } [TestCase(1)] [TestCase(2)] From b38a40d5a9633e20e22c1f2a2f95f0bff2db32ef Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 20:35:10 +0200 Subject: [PATCH 015/148] cosmetic --- .../TxBroadcasterTests.cs | 27 +++++-------------- 1 file changed, 6 insertions(+), 21 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 7904c5d3c71..70e18017c81 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -111,13 +111,8 @@ public async Task should_not_broadcast_persisted_tx_to_peer_too_quickly() peer.Received(2).SendNewTransactions(Arg.Any>(), true); } - [TestCase(1)] - [TestCase(2)] - [TestCase(99)] - [TestCase(100)] - [TestCase(101)] - [TestCase(1000)] - public void should_pick_best_persistent_txs_to_broadcast(int threshold) + [Test] + public void should_pick_best_persistent_txs_to_broadcast([Values(1, 2, 99, 100, 101, 1000)] int threshold) { _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); @@ -222,13 +217,8 @@ public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast( expectedHashes.Should().BeEquivalentTo(pickedHashes); } - [TestCase(1)] - [TestCase(2)] - [TestCase(99)] - [TestCase(100)] - [TestCase(101)] - [TestCase(1000)] - public void should_not_pick_txs_with_GasPrice_lower_than_CurrentBaseFee(int threshold) + [Test] + public void should_not_pick_txs_with_GasPrice_lower_than_CurrentBaseFee([Values(1, 2, 99, 100, 101, 1000)] int threshold) { _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); @@ -271,13 +261,8 @@ public void should_not_pick_txs_with_GasPrice_lower_than_CurrentBaseFee(int thre expectedTxs.Should().BeEquivalentTo(pickedTxs); } - [TestCase(1)] - [TestCase(2)] - [TestCase(99)] - [TestCase(100)] - [TestCase(101)] - [TestCase(1000)] - public void should_not_pick_1559_txs_with_MaxFeePerGas_lower_than_CurrentBaseFee(int threshold) + [Test] + public void should_not_pick_1559_txs_with_MaxFeePerGas_lower_than_CurrentBaseFee([Values(1, 2, 99, 100, 101, 1000)] int threshold) { _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); From 7f3df36ff87cff7214be29db45b6a2cd901c96dd Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 20:54:50 +0200 Subject: [PATCH 016/148] add broadcaster test --- .../Nethermind.TxPool.Test/TxBroadcasterTests.cs | 14 ++++++++++++++ .../Collections/BlobTxDistinctSortedPool.cs | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 70e18017c81..219cce75117 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -148,6 +148,20 @@ public void should_pick_best_persistent_txs_to_broadcast([Values(1, 2, 99, 100, expectedTxs.Should().BeEquivalentTo(pickedTxs); } + [Test] + public void should_not_add_blob_txs_to_persistent_txs([Values(true, false)] bool isBlob) + { + _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + + Transaction tx = Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .SignedAndResolved().TestObject; + + _broadcaster.Broadcast(tx, true); + _broadcaster.GetSnapshot().Length.Should().Be(isBlob ? 0 : 1); + } + [Test] public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast([Values(1, 2, 25, 50, 99, 100, 101, 1000)] int threshold) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 1ac9c139d50..d3014c612a4 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -72,7 +72,7 @@ private Transaction CreateLightTx(Transaction fullBlobTx) public IEnumerable GetBlobTransactions() { - // to refactor - it must enumerable starting from the best + // to refactor - it must lazy return starting from the best foreach (Transaction lightBlobTx in GetSnapshot()) { TryGetValue(lightBlobTx.Hash!, out Transaction? fullBlobTx); From a3d86d7c4a29d3b7dd96b87c8a077a21396d1c5b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 14 Jul 2023 21:03:30 +0200 Subject: [PATCH 017/148] add todos --- .../Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs | 1 + src/Nethermind/Nethermind.TxPool/TxPool.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index e5dd9dae123..4fcb44f32f9 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -23,6 +23,7 @@ public int Compare(Transaction? x, Transaction? y) // always allow replacement of zero fee txs if (y.MaxFeePerGas.IsZero) return -1; //ToDo: do we need it? + // ToDo: handle overflows if (y.MaxFeePerGas * 2 > x.MaxFeePerGas) return 1; if (y.MaxPriorityFeePerGas * 2 > x.MaxPriorityFeePerGas) return 1; if (y.MaxFeePerDataGas * 2 > x.MaxFeePerDataGas) return 1; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e26d667a0e5..5343b7b0b23 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -457,6 +457,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe } else { + // ToDo: add if (MaxFeePerDataGas < current) yield return (tx, UInt256.Zero) previousTxBottleneck ??= tx.CalculateAffordableGasPrice(_specProvider.GetCurrentHeadSpec().IsEip1559Enabled, _headInfo.CurrentBaseFee, balance); From 109cca4fb256e0c5972815074bb88be129cd7193 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 17 Jul 2023 19:07:35 +0200 Subject: [PATCH 018/148] add MaxFeePerDataGas check when updating gas bottleneck --- .../Nethermind.Blockchain/ChainHeadInfoProvider.cs | 7 +++++++ .../Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs | 6 ++++-- src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs | 2 ++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 7 +++++-- 4 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs index a2701230e2d..03cfefdfdda 100644 --- a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs +++ b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs @@ -6,6 +6,7 @@ using Nethermind.Blockchain.Spec; using Nethermind.Core; using Nethermind.Core.Specs; +using Nethermind.Evm; using Nethermind.Int256; using Nethermind.State; using Nethermind.TxPool; @@ -42,12 +43,18 @@ public ChainHeadInfoProvider(IChainHeadSpecProvider specProvider, IBlockTree blo public UInt256 CurrentBaseFee { get; private set; } + public UInt256 CurrentPricePerDataGas { get; internal set; } + public event EventHandler? HeadChanged; private void OnHeadChanged(object? sender, BlockReplacementEventArgs e) { BlockGasLimit = e.Block!.GasLimit; CurrentBaseFee = e.Block.Header.BaseFeePerGas; + CurrentPricePerDataGas = + DataGasCalculator.TryCalculateDataGasPricePerUnit(e.Block.Header, out UInt256 currentPricePerDataGas) + ? currentPricePerDataGas + : UInt256.Zero; HeadChanged?.Invoke(sender, e); } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index 4fcb44f32f9..9e5267c6bed 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -6,11 +6,14 @@ namespace Nethermind.TxPool.Comparison; +/// +/// Compare fee of newcomer blob transaction with fee of transaction intended to be replaced +/// public class CompareReplacedBlobTx : IComparer { public static readonly CompareReplacedBlobTx Instance = new(); - public CompareReplacedBlobTx() { } + private CompareReplacedBlobTx() { } // To replace old blob transaction, new transaction needs to have fee at least 2x higher than current fee. // 2x higher must be MaxPriorityFeePerGas, MaxFeePerGas and MaxFeePerDataGas @@ -23,7 +26,6 @@ public int Compare(Transaction? x, Transaction? y) // always allow replacement of zero fee txs if (y.MaxFeePerGas.IsZero) return -1; //ToDo: do we need it? - // ToDo: handle overflows if (y.MaxFeePerGas * 2 > x.MaxFeePerGas) return 1; if (y.MaxPriorityFeePerGas * 2 > x.MaxPriorityFeePerGas) return 1; if (y.MaxFeePerDataGas * 2 > x.MaxFeePerDataGas) return 1; diff --git a/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs b/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs index ce089c74908..0ba937d8169 100644 --- a/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs +++ b/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs @@ -18,6 +18,8 @@ public interface IChainHeadInfoProvider public UInt256 CurrentBaseFee { get; } + public UInt256 CurrentPricePerDataGas { get; } + event EventHandler HeadChanged; } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 5343b7b0b23..7e34a19eea6 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -457,11 +457,14 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe } else { - // ToDo: add if (MaxFeePerDataGas < current) yield return (tx, UInt256.Zero) previousTxBottleneck ??= tx.CalculateAffordableGasPrice(_specProvider.GetCurrentHeadSpec().IsEip1559Enabled, _headInfo.CurrentBaseFee, balance); - if (tx.Nonce == currentNonce + i) + if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) + { + yield return (tx, UInt256.Zero); + } + else if (tx.Nonce == currentNonce + i) { UInt256 effectiveGasPrice = tx.CalculateEffectiveGasPrice(_specProvider.GetCurrentHeadSpec().IsEip1559Enabled, From 0b8f72d8486255288a468cf44086adc7381680a8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 17 Jul 2023 19:08:20 +0200 Subject: [PATCH 019/148] regression test for MaxFeePerDataGas check --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 35 +++++++++++++++++++ .../Collections/BlobTxDistinctSortedPool.cs | 4 +++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 +++ 3 files changed, 43 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index f6b9c3c054d..80830b7d169 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1838,6 +1838,41 @@ public void should_remove_replaced_blob_tx() .Excluding(t => t.PoolIndex)); // ...and PoolIndex } + [Test] + public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_lower_than_current([Values(true, false)] bool isBlob) + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + _headInfo.CurrentPricePerDataGas = UInt256.MaxValue; + + Transaction tx = Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(UInt256.One) + .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerDataGas(isBlob ? UInt256.One : null) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(isBlob ? 1 : 0); + _txPool.GetPendingTransactionsCount().Should().Be(isBlob ? 0 : 1); + if (isBlob) + { + _txPool.TryGetLightBlobTransaction(tx.Hash!, out Transaction blobTxReturned); + blobTxReturned.Should().NotBeEquivalentTo(tx); + blobTxReturned.GasBottleneck.Should().Be(UInt256.Zero); + } + else + { + _txPool.TryGetPendingTransaction(tx.Hash!, out Transaction eip1559tx); + eip1559tx.Should().BeEquivalentTo(tx); + eip1559tx.GasBottleneck.Should().Be(UInt256.One); + } + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index d3014c612a4..c5afadcc31b 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -100,6 +100,10 @@ public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) return false; } + // only for tests - to test sorting + internal bool TryGetLightValue(Keccak hash, out Transaction? lightBlobTx) + => base.TryGetValue(hash, out lightBlobTx); + protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 7e34a19eea6..ac86cb154d1 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -590,6 +590,10 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) } } + // only for tests - to test sorting + internal bool TryGetLightBlobTransaction(Keccak hash, out Transaction? transaction) + => _blobTransactions.TryGetLightValue(hash, out transaction); + // should own transactions (in broadcaster) be also checked here? // maybe it should use NonceManager, as it already has info about local txs? public UInt256 GetLatestPendingNonce(Address address) From 5a204868ce511beeeecfcd46bd82e740e8f09e1c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 17 Jul 2023 19:17:29 +0200 Subject: [PATCH 020/148] cosmetic --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 80830b7d169..08c076e3734 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -770,7 +770,7 @@ public void should_calculate_gasBottleneck_properly() } [Test] - public async Task should_remove_GasBottleneck_of_old_nonces() + public async Task should_remove_txs_with_old_nonces_when_updating_GasBottleneck() { _txPool = CreatePool(); Transaction[] transactions = new Transaction[5]; @@ -793,6 +793,7 @@ public async Task should_remove_GasBottleneck_of_old_nonces() } await RaiseBlockAddedToMainAndWaitForTransactions(5); + _txPool.GetPendingTransactionsCount().Should().Be(2); _txPool.GetPendingTransactions().Count(t => t.GasBottleneck == 0).Should().Be(0); _txPool.GetPendingTransactions().Max(t => t.GasBottleneck).Should().Be((UInt256)5); } From a6e1e16c70604026c7adabac9d42eb01807b13d2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 11:50:16 +0200 Subject: [PATCH 021/148] unify & simplify tests --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 78 +++++-------------- 1 file changed, 21 insertions(+), 57 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 08c076e3734..ce6ef2edd47 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1611,70 +1611,39 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g } [Test] - public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool() + public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) { _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(UInt256.One) .WithMaxPriorityFeePerGas(UInt256.One) .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; Transaction nonceGapBlobTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) - .WithNonce((UInt256)2) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.NonceGap); - } - - [Test] - public void should_add_nonce_gap_tx_to_not_full_TxPool_if_is_not_blob_type() - { - _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction firstTx = Build.A.Transaction - .WithType(TxType.EIP1559) - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - Transaction nonceGap1559Tx = Build.A.Transaction - .WithType(TxType.EIP1559) + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(UInt256.One) .WithMaxPriorityFeePerGas(UInt256.One) .WithNonce((UInt256)2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(nonceGap1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(isBlob ? AcceptTxResult.NonceGap : AcceptTxResult.Accepted); } [Test] public void should_not_allow_to_have_pending_transactions_of_both_blob_type_and_other([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) { - Transaction GetBlobTx(UInt256 nonce) - { - return Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) - .WithNonce(nonce) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - } - - Transaction Get1559Tx(UInt256 nonce) + Transaction GetTx(bool isBlob, UInt256 nonce) { return Build.A.Transaction - .WithType(TxType.EIP1559) + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(UInt256.One) .WithMaxPriorityFeePerGas(UInt256.One) .WithNonce(nonce) @@ -1684,8 +1653,8 @@ Transaction Get1559Tx(UInt256 nonce) _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - Transaction firstTx = firstIsBlob ? GetBlobTx(UInt256.Zero) : Get1559Tx(UInt256.Zero); - Transaction secondTx = secondIsBlob ? GetBlobTx(UInt256.One) : Get1559Tx(UInt256.One); + Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); + Transaction secondTx = GetTx(secondIsBlob, UInt256.One); _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted); @@ -1694,20 +1663,11 @@ Transaction Get1559Tx(UInt256 nonce) [Test] public async Task should_allow_to_have_pending_transaction_of_other_type_if_conflicting_one_was_included([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) { - Transaction GetBlobTx(UInt256 nonce) + Transaction GetTx(bool isBlob, UInt256 nonce) { return Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) - .WithNonce(nonce) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - } - - Transaction Get1559Tx(UInt256 nonce) - { - return Build.A.Transaction - .WithType(TxType.EIP1559) + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(UInt256.One) .WithMaxPriorityFeePerGas(UInt256.One) .WithNonce(nonce) @@ -1717,8 +1677,8 @@ Transaction Get1559Tx(UInt256 nonce) _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - Transaction firstTx = firstIsBlob ? GetBlobTx(UInt256.Zero) : Get1559Tx(UInt256.Zero); - Transaction secondTx = secondIsBlob ? GetBlobTx(UInt256.One) : Get1559Tx(UInt256.One); + Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); + Transaction secondTx = GetTx(secondIsBlob, UInt256.One); _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); @@ -1730,12 +1690,16 @@ Transaction Get1559Tx(UInt256 nonce) _txPool.GetPendingTransactionsCount().Should().Be(0); _txPool.GetPendingBlobTransactionsCount().Should().Be(0); _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingTransactionsCount().Should().Be(secondIsBlob ? 0 : 1); + _txPool.GetPendingBlobTransactionsCount().Should().Be(secondIsBlob ? 1 : 0); } + // blob collection is designed to be infinite, so there is no point in checking if is full and comparing + // incoming tx with the worst already pending one (candidate for evicting). There will be no eviction. [Test] public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() { - TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); From 6596c1d13892b300a5f3a8a8d0fca20baf0fe127 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 11:50:32 +0200 Subject: [PATCH 022/148] cosmetics in BlobTxStorage --- .../Nethermind.TxPool/BlobTxStorage.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index d9d64135cc5..145d3ca9c82 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -21,17 +21,6 @@ public BlobTxStorage(IDb database) public bool TryGet(Keccak hash, out Transaction? transaction) => TryDecode(_database.Get(hash), out transaction); - public IEnumerable GetAll() - { - foreach (byte[] txBytes in _database.GetAllValues()) - { - if (TryDecode(txBytes, out Transaction? transaction)) - { - yield return transaction!; - } - } - } - private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) { if (txBytes is not null) @@ -51,6 +40,17 @@ private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) return false; } + public IEnumerable GetAll() + { + foreach (byte[] txBytes in _database.GetAllValues()) + { + if (TryDecode(txBytes, out Transaction? transaction)) + { + yield return transaction!; + } + } + } + public void Add(Transaction transaction) { if (transaction == null || transaction.Hash == null) From 07870f1eb508f2ac106702544be7c75b72a113c2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 13:07:49 +0200 Subject: [PATCH 023/148] add AcceptTxResult dedicated for rejected replacements --- src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index e7258ef2695..c24478ec770 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -76,10 +76,15 @@ namespace Nethermind.TxPool /// public static readonly AcceptTxResult SenderIsContract = new(12, nameof(SenderIsContract)); + /// + /// Transaction is not allowed to replace the one already in the pool. Fee bump is too low or some requirements are not fulfilled + /// + public static readonly AcceptTxResult ReplacementNotAllowed = new(13, nameof(ReplacementNotAllowed)); + /// /// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs /// - public static readonly AcceptTxResult PendingTxsOfOtherType = new(13, nameof(PendingTxsOfOtherType)); + public static readonly AcceptTxResult PendingTxsOfOtherType = new(14, nameof(PendingTxsOfOtherType)); private int Id { get; } private string Code { get; } From 73872e1da9c5b70f09afe30392bb8f84a3fe3afa Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 13:08:34 +0200 Subject: [PATCH 024/148] adjust tests, add test for blob tx replacements --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 38 ++++++++++++++++++- 1 file changed, 36 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index ce6ef2edd47..06b293e433c 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1011,7 +1011,7 @@ public void should_delete_pending_transactions() } [Test] - public void should_return_feeTooLowTooCompete_result_when_trying_to_send_transaction_with_same_nonce_for_same_address() + public void should_return_ReplacementNotAllowed_when_trying_to_send_transaction_with_same_nonce_and_same_fee_for_same_address() { _txPool = CreatePool(); var result1 = _txPool.SubmitTx(GetTransaction(TestItem.PrivateKeyA, TestItem.AddressA), TxHandlingOptions.PersistentBroadcast | TxHandlingOptions.ManagedNonce); @@ -1019,7 +1019,7 @@ public void should_return_feeTooLowTooCompete_result_when_trying_to_send_transac _txPool.GetOwnPendingTransactions().Length.Should().Be(1); _txPool.GetPendingTransactionsCount().Should().Be(1); var result2 = _txPool.SubmitTx(GetTransaction(TestItem.PrivateKeyA, TestItem.AddressB), TxHandlingOptions.PersistentBroadcast | TxHandlingOptions.ManagedNonce); - result2.Should().Be(AcceptTxResult.FeeTooLowToCompete); + result2.Should().Be(AcceptTxResult.ReplacementNotAllowed); _txPool.GetOwnPendingTransactions().Length.Should().Be(1); _txPool.GetPendingTransactionsCount().Should().Be(1); } @@ -1838,6 +1838,40 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ } } + [Test] + public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, 2, 3, 4, 5, 6)] int blobsInFirstTx, [Values(1, 2, 3, 4, 5, 6)] int blobsInSecondTx) + { + bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; + const int initialFee = 10; + + _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(blobsInFirstTx) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(initialFee) + .WithMaxPriorityFeePerGas(initialFee) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction secondTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(blobsInSecondTx) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(firstTx.MaxFeePerGas * 2) + .WithMaxPriorityFeePerGas(firstTx.MaxPriorityFeePerGas * 2) + .WithMaxFeePerDataGas(firstTx.MaxFeePerDataGas * 2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(shouldReplace ? AcceptTxResult.Accepted : AcceptTxResult.ReplacementNotAllowed); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(shouldReplace ? secondTx.Hash! : firstTx.Hash!, out Transaction tx).Should().BeTrue(); + tx.Should().BeEquivalentTo(shouldReplace ? secondTx : firstTx); + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] From b11a27dc5f743f2ee09ae31c081ee2be501707d5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 13:08:55 +0200 Subject: [PATCH 025/148] simplify TxPool --- .../Collections/BlobTxDistinctSortedPool.cs | 6 +-- .../Comparison/CompareReplacedBlobTx.cs | 4 ++ src/Nethermind/Nethermind.TxPool/Metrics.cs | 7 +++- src/Nethermind/Nethermind.TxPool/TxPool.cs | 37 +++++++++++-------- 4 files changed, 32 insertions(+), 22 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index c5afadcc31b..89bdfa0ba1b 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -40,9 +40,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) { - Transaction lightBlobTx = CreateLightTx(fullBlobTx); - - if (base.TryInsert(fullBlobTx.Hash, lightBlobTx, out removed)) + if (base.TryInsert(fullBlobTx.Hash, CreateLightTx(fullBlobTx), out removed)) { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); _blobTxStorage.Add(fullBlobTx); @@ -72,7 +70,7 @@ private Transaction CreateLightTx(Transaction fullBlobTx) public IEnumerable GetBlobTransactions() { - // to refactor - it must lazy return starting from the best + // ToDo: to refactor - it must lazy enumerate starting from the best foreach (Transaction lightBlobTx in GetSnapshot()) { TryGetValue(lightBlobTx.Hash!, out Transaction? fullBlobTx); diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index 9e5267c6bed..d708f92e275 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -23,6 +23,10 @@ public int Compare(Transaction? x, Transaction? y) if (ReferenceEquals(null, y)) return 1; if (ReferenceEquals(null, x)) return -1; + // do not allow to replace blob tx by the one with lower number of blobs + if (y.BlobVersionedHashes is null || x.BlobVersionedHashes is null) return 1; + if (y.BlobVersionedHashes.Length > x.BlobVersionedHashes.Length) return 1; + // always allow replacement of zero fee txs if (y.MaxFeePerGas.IsZero) return -1; //ToDo: do we need it? diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 609da4987c9..7a635ae0a13 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -60,10 +60,13 @@ public static class Metrics public static long PendingTransactionsGasLimitTooHigh { get; set; } [CounterMetric] - [Description( - "Number of pending transactions received that were ignored after passing early rejections as balance is too low to compete with lowest effective fee in transaction pool.")] + [Description("Number of pending transactions received that were ignored after passing early rejections as balance is too low to compete with lowest effective fee in transaction pool.")] public static long PendingTransactionsPassedFiltersButCannotCompeteOnFees { get; set; } + [CounterMetric] + [Description("Number of pending transactions received that were trying to replace tx with the same sender and nonce and failed.")] + public static long PendingTransactionsPassedFiltersButCannotReplace { get; set; } + [CounterMetric] [Description("Number of pending transactions that reached filters which are resource expensive")] public static long PendingTransactionsWithExpensiveFiltering { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ac86cb154d1..02a2ae56a78 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -388,25 +388,16 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe bool inserted = (tx.SupportsBlobs ? _blobTransactions.TryInsert(tx, out Transaction? removed) : _transactions.TryInsert(tx.Hash!, tx, out removed)); - if (inserted && tx.Hash != removed?.Hash) - { - (tx.SupportsBlobs ? _blobTransactions : _transactions).UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); - Metrics.PendingTransactionsAdded++; - if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } - if (removed is not null) - { - EvictedPending?.Invoke(this, new TxEventArgs(removed)); - // transaction which was on last position in sorted TxPool and was deleted to give - // a place for a newly added tx (with higher priority) is now removed from hashCache - // to give it opportunity to come back to TxPool in the future, when fees drops - _hashCache.DeleteFromLongTerm(removed.Hash!); - Metrics.PendingTransactionsEvicted++; - } + if (!inserted) + { + Metrics.PendingTransactionsPassedFiltersButCannotReplace++; + return AcceptTxResult.ReplacementNotAllowed; } - else + + if (tx.Hash == removed?.Hash) { - if (isPersistentBroadcast && inserted) + if (isPersistentBroadcast) { // it means it was added and immediately evicted - we are adding only to persistent broadcast _broadcaster.Broadcast(tx, isPersistentBroadcast); @@ -414,6 +405,20 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; return AcceptTxResult.FeeTooLowToCompete; } + + (tx.SupportsBlobs ? _blobTransactions : _transactions).UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); + Metrics.PendingTransactionsAdded++; + if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } + + if (removed is not null) + { + EvictedPending?.Invoke(this, new TxEventArgs(removed)); + // transaction which was on last position in sorted TxPool and was deleted to give + // a place for a newly added tx (with higher priority) is now removed from hashCache + // to give it opportunity to come back to TxPool in the future, when fees drops + _hashCache.DeleteFromLongTerm(removed.Hash!); + Metrics.PendingTransactionsEvicted++; + } } _broadcaster.Broadcast(tx, isPersistentBroadcast); From 5da94880a6d5a3a0d072ae96f54f4b1eb8d00ada Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 13:10:22 +0200 Subject: [PATCH 026/148] refactor to reuse id of deprecated result --- src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index c24478ec770..941a73f4b8a 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -66,25 +66,19 @@ namespace Nethermind.TxPool public static readonly AcceptTxResult OldNonce = new(10, nameof(OldNonce)); /// - /// A transaction with same nonce has been signed locally already and is awaiting in the pool. - /// (I would like to change this behaviour to allow local replacement) + /// Transaction is not allowed to replace the one already in the pool. Fee bump is too low or some requirements are not fulfilled /// - public static readonly AcceptTxResult OwnNonceAlreadyUsed = new(11, nameof(OwnNonceAlreadyUsed)); + public static readonly AcceptTxResult ReplacementNotAllowed = new(11, nameof(ReplacementNotAllowed)); /// /// Transaction sender has code hash that is not null. /// public static readonly AcceptTxResult SenderIsContract = new(12, nameof(SenderIsContract)); - /// - /// Transaction is not allowed to replace the one already in the pool. Fee bump is too low or some requirements are not fulfilled - /// - public static readonly AcceptTxResult ReplacementNotAllowed = new(13, nameof(ReplacementNotAllowed)); - /// /// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs /// - public static readonly AcceptTxResult PendingTxsOfOtherType = new(14, nameof(PendingTxsOfOtherType)); + public static readonly AcceptTxResult PendingTxsOfOtherType = new(13, nameof(PendingTxsOfOtherType)); private int Id { get; } private string Code { get; } From 3071be34805a8554babcdac7a7820223001c54b1 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 13:23:28 +0200 Subject: [PATCH 027/148] simplify filters --- .../Nethermind.TxPool/Filters/BalanceTooLowFilter.cs | 5 +++-- .../Nethermind.TxPool/Filters/BalanceZeroFilter.cs | 5 +---- src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs | 3 +++ 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs index 52dfe150ff5..dc8e0925519 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/BalanceTooLowFilter.cs @@ -37,8 +37,9 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO UInt256 cumulativeCost = UInt256.Zero; bool overflow = false; Transaction[] sameTypeTxs = tx.SupportsBlobs - ? _blobTxs.GetBucketSnapshot(tx.SenderAddress!) - : _txs.GetBucketSnapshot(tx.SenderAddress!); // since unknownSenderFilter will run before this one + ? _blobTxs.GetBucketSnapshot(tx.SenderAddress!) // it will create a snapshot of light txs (without actual blobs) + : _txs.GetBucketSnapshot(tx.SenderAddress!); + // tx.SenderAddress! as unknownSenderFilter will run before this one for (int i = 0; i < sameTypeTxs.Length; i++) { diff --git a/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs index 81fd6736444..30d457dd049 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/BalanceZeroFilter.cs @@ -43,10 +43,7 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO AcceptTxResult.InsufficientFunds.WithMessage($"Balance is {balance} less than sending value {tx.Value}"); } - // data gas cost is not added here, so in case of overflow at that step, blob transactions will be rejected - // later, in BalanceTooLowFilter. It's rare scenario, so probably it's not worth to complicate code here - if (UInt256.MultiplyOverflow(tx.MaxFeePerGas, (UInt256)tx.GasLimit, out UInt256 txCostAndValue) || - UInt256.AddOverflow(txCostAndValue, tx.Value, out txCostAndValue)) + if (tx.IsOverflowInTxCostAndValue(out UInt256 txCostAndValue)) { Metrics.PendingTransactionsBalanceBelowValue++; if (_logger.IsTrace) diff --git a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs index dc5b0eb3603..fe0d6cb845f 100644 --- a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs @@ -81,5 +81,8 @@ internal static bool IsOverflowWhenAddingTxCostToCumulative(this Transaction tx, return overflow; } + + internal static bool IsOverflowInTxCostAndValue(this Transaction tx, out UInt256 txCost) + => IsOverflowWhenAddingTxCostToCumulative(tx, UInt256.Zero, out txCost); } } From 9cc03cce783519b8db96ac5278a182471ab72612 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 14:01:57 +0200 Subject: [PATCH 028/148] add configurable sizes of blob pool and blob cache --- .../Collections/BlobTxDistinctSortedPool.cs | 10 ++++++---- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 6 ++++++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 11 +++++++++-- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 2 ++ 4 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 89bdfa0ba1b..b54ee0c9f7f 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Caching; @@ -12,13 +13,14 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { private readonly ITxStorage _blobTxStorage; - private readonly LruCache _blobTxCache = new(256, 256, "blob txs cache"); + private readonly LruCache _blobTxCache; private readonly ILogger _logger; - public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, int capacity, IComparer comparer, ILogManager logManager) - : base(capacity, comparer, logManager) + public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfig txPoolConfig, IComparer comparer, ILogManager logManager) + : base(txPoolConfig.BlobPoolSize, comparer, logManager) { - _blobTxStorage = blobTxStorage; + _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); + _blobTxCache = new(txPoolConfig.BlobCacheSize, txPoolConfig.BlobCacheSize, "blob txs cache"); _logger = logManager.GetClassLogger(); RecreateLightTxCollectionAndCache(blobTxStorage); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index e46f9bc79d4..289fce44f8e 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -13,6 +13,12 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "Max number of transactions held in mempool (more transactions in mempool mean more memory used")] int Size { get; set; } + [ConfigItem(DefaultValue = "131072", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] + int BlobPoolSize { get; set; } + + [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory (recent ones)")] + int BlobCacheSize { get; set; } + [ConfigItem(DefaultValue = "524288", Description = "Max number of cached hashes of already known transactions." + "It is set automatically by the memory hint.")] diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 02a2ae56a78..c494d0c7af2 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -100,7 +100,7 @@ public TxPool(IEthereumEcdsa ecdsa, _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); // we need some limit of blob txs, but extremely high to be limitless in practice - _blobTransactions = new BlobTxDistinctSortedPool(_blobTxStorage, 32 * MemoryAllowance.MemPoolSize, comparer, logManager); + _blobTransactions = new BlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _headInfo.HeadChanged += OnHeadChange; @@ -467,7 +467,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) { - yield return (tx, UInt256.Zero); + gasBottleneck = UInt256.Zero; } else if (tx.Nonce == currentNonce + i) { @@ -497,6 +497,13 @@ private void UpdateBuckets() if (_logger.IsWarn) _logger.Warn($"TxPool exceeds the config size {_transactions.Count}/{_txPoolConfig.Size}"); _transactions.UpdatePool(_accounts, _updateBucket); + // ensure the capacity of the blob pool + if (_blobTransactions.Count > _txPoolConfig.BlobPoolSize) + if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.BlobPoolSize}"); + + if (_blobTransactions.Count == _txPoolConfig.BlobPoolSize) + if (_logger.IsDebug) _logger.Debug($"Blob TxPool has reached max size of {_txPoolConfig.BlobPoolSize}, blob txs can be evicted now"); + _blobTransactions.UpdatePool(_accounts, _updateBucket); } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 7e08186f337..26e55136ce7 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,6 +7,8 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; + public int BlobPoolSize { get; set; } = 128 * 1024; + public int BlobCacheSize { get; set; } = 256; public int HashCacheSize { get; set; } = 512 * 1024; public long? GasLimit { get; set; } = null; public int? ReportMinutes { get; set; } = null; From 761a6c5d66c4ca9e473986cd129b2577b5a9c030 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 14:17:31 +0200 Subject: [PATCH 029/148] optimize TxType filter --- .../Nethermind.TxPool/Collections/SortedPool.cs | 3 +++ .../Nethermind.TxPool/Filters/TxTypeTxFilter.cs | 12 ++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 76988734ddb..024ecf719a4 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -390,6 +390,9 @@ protected virtual bool Remove(TKey key, TValue value) private void UpdateIsFull() => _isFull = _cacheMap.Count >= _capacity; + [MethodImpl(MethodImplOptions.Synchronized)] + public bool ContainsBucket(TGroupKey groupKey) => + _buckets.ContainsKey(groupKey); [MethodImpl(MethodImplOptions.Synchronized)] public bool TryGetBucket(TGroupKey groupKey, out TValue[] items) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs index 3eac786ff90..d0013828890 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs @@ -22,13 +22,9 @@ public TxTypeTxFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) { - if (tx.SupportsBlobs) - { - return _txs.TryGetBucket(tx.SenderAddress!, out _) ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted; - } - else - { - return _blobTxs.TryGetBucket(tx.SenderAddress!, out _) ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted; - } + return (tx.SupportsBlobs ? _txs : _blobTxs) + .ContainsBucket(tx.SenderAddress!) // as unknownSenderFilter will run before this one + ? AcceptTxResult.PendingTxsOfOtherType + : AcceptTxResult.Accepted; } } From 028f650ca68115d4295fe44aeb66c009ffd623bf Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 16:17:20 +0200 Subject: [PATCH 030/148] add light tx as separate class to reuse --- .../Collections/BlobTxDistinctSortedPool.cs | 22 ++------------- .../Nethermind.TxPool/LightTransaction.cs | 28 +++++++++++++++++++ 2 files changed, 30 insertions(+), 20 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/LightTransaction.cs diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index b54ee0c9f7f..7027418c0ad 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -33,7 +33,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) { - if (base.TryInsert(fullBlobTx.Hash, CreateLightTx(fullBlobTx))) + if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx))) { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); } @@ -42,7 +42,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) { - if (base.TryInsert(fullBlobTx.Hash, CreateLightTx(fullBlobTx), out removed)) + if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out removed)) { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); _blobTxStorage.Add(fullBlobTx); @@ -52,24 +52,6 @@ public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) return false; } - private Transaction CreateLightTx(Transaction fullBlobTx) - { - return new Transaction - { - Type = TxType.Blob, - Hash = fullBlobTx.Hash, - SenderAddress = fullBlobTx.SenderAddress, - Nonce = fullBlobTx.Nonce, - Value = fullBlobTx.Value, - GasLimit = fullBlobTx.GasLimit, - GasPrice = fullBlobTx.GasPrice, // means MaxPriorityFeePerGas - DecodedMaxFeePerGas = fullBlobTx.DecodedMaxFeePerGas, - MaxFeePerDataGas = fullBlobTx.MaxFeePerDataGas, - BlobVersionedHashes = new byte[fullBlobTx.BlobVersionedHashes!.Length][], - GasBottleneck = fullBlobTx.GasBottleneck, - }; - } - public IEnumerable GetBlobTransactions() { // ToDo: to refactor - it must lazy enumerate starting from the best diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs new file mode 100644 index 00000000000..75578d679de --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -0,0 +1,28 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + + +using Nethermind.Core; + +namespace Nethermind.TxPool; + +/// +/// For sorting reasons - without storing full, large txs in memory +/// +public class LightTransaction : Transaction +{ + public LightTransaction(Transaction fullTx) + { + Type = TxType.Blob; + Hash = fullTx.Hash; + SenderAddress = fullTx.SenderAddress; + Nonce = fullTx.Nonce; + Value = fullTx.Value; + GasLimit = fullTx.GasLimit; + GasPrice = fullTx.GasPrice; // means MaxPriorityFeePerGas + DecodedMaxFeePerGas = fullTx.DecodedMaxFeePerGas; + MaxFeePerDataGas = fullTx.MaxFeePerDataGas; + BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; + GasBottleneck = fullTx.GasBottleneck; + } +} From 1b479214014b0924c688221ff95b82945ff608e3 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 16:19:07 +0200 Subject: [PATCH 031/148] dont return blob txs from broadcaster and other refactorings --- .../TransactionSelectorTests.cs | 2 +- .../Producers/TxPoolTxSource.cs | 2 +- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 4 ++-- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 +- .../Nethermind.TxPool/TxBroadcaster.cs | 18 ++++++++++++------ src/Nethermind/Nethermind.TxPool/TxPool.cs | 10 ++++++---- .../Nethermind.TxPool/TxPoolInfoProvider.cs | 2 ++ 7 files changed, 25 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index 7faa0527bc5..f9d8de672ec 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -243,7 +243,7 @@ void SetAccountStates(IEnumerable
missingAddresses) .Where(t => t.SupportsBlobs) .ToArray(); transactionPool.GetPendingTransactionsBySender().Returns(transactions); - transactionPool.GetBlobTransactions().Returns(blobTransactions); + transactionPool.GetPendingBlobTransactions().Returns(blobTransactions); BlocksConfig blocksConfig = new() { MinGasPrice = testCase.MinGasPriceForMining }; ITxFilterPipeline txFilterPipeline = new TxFilterPipelineBuilder(LimboLogs.Instance) .WithMinGasPriceFilter(blocksConfig, specProvider) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index f7051478c70..f0afa9eb19b 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -53,7 +53,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi .ThenBy(ByHashTxComparer.Instance); // in order to sort properly and not lose transactions we need to differentiate on their identity which provided comparer might not be doing IEnumerable transactions = GetOrderedTransactions(pendingTransactions, comparer); - IEnumerable blobTransactions = _transactionPool.GetBlobTransactions(); + IEnumerable blobTransactions = _transactionPool.GetPendingBlobTransactions(); if (_logger.IsDebug) _logger.Debug($"Collecting pending transactions at block gas limit {gasLimit}."); int selectedTransactions = 0; diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 68846987ca5..96979608ea7 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -21,10 +21,10 @@ public interface ITxPool IDictionary GetPendingTransactionsBySender(); /// - /// Lazy return blob txs starting from the best one + /// Lazy enumerate blob txs starting from the best one /// /// - IEnumerable GetBlobTransactions(); + IEnumerable GetPendingBlobTransactions(); /// /// from a specific sender, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index df3337154fc..30f9c4e5f51 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -25,7 +25,7 @@ private NullTxPool() { } public IDictionary GetPendingTransactionsBySender() => new Dictionary(); - public IEnumerable GetBlobTransactions() => Array.Empty(); + public IEnumerable GetPendingBlobTransactions() => Array.Empty(); public void AddPeer(ITxPoolPeer peer) { } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index dfe2a0761d5..d07be23977f 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -37,9 +37,9 @@ internal class TxBroadcaster : IDisposable private readonly ConcurrentDictionary _peers = new(); /// - /// Transactions published locally (initiated by this node users) or reorganised. + /// Transactions published locally (initiated by this node users). /// - private readonly SortedPool _persistentTxs; + private readonly TxDistinctSortedPool _persistentTxs; /// /// Transactions added by external peers between timer elapses. @@ -81,6 +81,7 @@ public TxBroadcaster(IComparer comparer, _timer.Start(); } + // only for testing reasons internal Transaction[] GetSnapshot() => _persistentTxs.GetSnapshot(); public void Broadcast(Transaction tx, bool isPersistent) @@ -98,10 +99,9 @@ public void Broadcast(Transaction tx, bool isPersistent) private void StartBroadcast(Transaction tx) { NotifyPeersAboutLocalTx(tx); - if (tx.Hash is not null && !tx.SupportsBlobs) + if (tx.Hash is not null) { - // somehow save only hashes of persistent blob txs? Or not add them here at all? Not add at all for now - _persistentTxs.TryInsert(tx.Hash, tx); + _persistentTxs.TryInsert(tx.Hash, tx.SupportsBlobs ? new LightTransaction(tx) : tx); } } @@ -292,7 +292,13 @@ private void NotifyPeersAboutLocalTx(Transaction tx) public bool TryGetPersistentTx(Keccak hash, out Transaction? transaction) { - return _persistentTxs.TryGetValue(hash, out transaction); + if (_persistentTxs.TryGetValue(hash, out transaction) && !transaction.SupportsBlobs) + { + return true; + } + + transaction = default; + return false; } public bool AddPeer(ITxPoolPeer peer) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index c494d0c7af2..7469ae14ec4 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -158,13 +158,13 @@ public IDictionary GetPendingTransactionsBySender() => public Transaction[] GetPendingTransactionsBySender(Address address) => _transactions.GetBucketSnapshot(address); + // only for testing reasons internal Transaction[] GetOwnPendingTransactions() => _broadcaster.GetSnapshot(); - // public Transaction[] GetPendingBlobTransactions() => _blobTransactions.GetSnapshot(); + public IEnumerable GetPendingBlobTransactions() => _blobTransactions.GetBlobTransactions(); public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - public IEnumerable GetBlobTransactions() => _blobTransactions.GetBlobTransactions(); // public Transaction[] GetPendingBlobTransactionsBySender(Address address) => // _blobTransactions.GetBucketSnapshot(address); @@ -574,6 +574,8 @@ public bool RemoveTransaction(Keccak? hash) hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction); if (!hasBeenRemoved) { + // add flag keepInDb if coming from processed block? and remove after finalization? + // actually for blobs it is removed here only after block processing hasBeenRemoved = _blobTransactions.TryRemove(hash, out transaction); } @@ -597,8 +599,8 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) lock (_locker) { return _transactions.TryGetValue(hash, out transaction) - || _broadcaster.TryGetPersistentTx(hash, out transaction) - || _blobTransactions.TryGetValue(hash, out transaction); + || _blobTransactions.TryGetValue(hash, out transaction) + || _broadcaster.TryGetPersistentTx(hash, out transaction); } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolInfoProvider.cs b/src/Nethermind/Nethermind.TxPool/TxPoolInfoProvider.cs index 198a130ee1a..d1034581fca 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolInfoProvider.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolInfoProvider.cs @@ -21,6 +21,8 @@ public TxPoolInfoProvider(IAccountStateProvider accountStateProvider, ITxPool tx public TxPoolInfo GetInfo() { + // only std txs are picked here. Should we add blobs? + // BTW this class should be rewritten or removed - a lot of unnecessary allocations var groupedTransactions = _txPool.GetPendingTransactionsBySender(); var pendingTransactions = new Dictionary>(); var queuedTransactions = new Dictionary>(); From 8145f448d88d612c1a4d5314245803327d6afa37 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 16:21:32 +0200 Subject: [PATCH 032/148] add test for broadcaster --- .../TxBroadcasterTests.cs | 73 ++++++++++++++++++- 1 file changed, 71 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 219cce75117..2ab602141db 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -149,7 +149,7 @@ public void should_pick_best_persistent_txs_to_broadcast([Values(1, 2, 99, 100, } [Test] - public void should_not_add_blob_txs_to_persistent_txs([Values(true, false)] bool isBlob) + public void should_add_light_form_of_blob_txs_to_persistent_txs_but_not_return_if_requested([Values(true, false)] bool isBlob) { _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); @@ -159,7 +159,11 @@ public void should_not_add_blob_txs_to_persistent_txs([Values(true, false)] bool .SignedAndResolved().TestObject; _broadcaster.Broadcast(tx, true); - _broadcaster.GetSnapshot().Length.Should().Be(isBlob ? 0 : 1); + _broadcaster.GetSnapshot().Length.Should().Be(1); + _broadcaster.GetSnapshot()[0].Should().BeEquivalentTo(isBlob ? new LightTransaction(tx) : tx); + + _broadcaster.TryGetPersistentTx(tx.Hash, out Transaction returnedTx).Should().Be(!isBlob); + returnedTx.Should().BeEquivalentTo(isBlob ? null : tx); } [Test] @@ -231,6 +235,71 @@ public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast( expectedHashes.Should().BeEquivalentTo(pickedHashes); } + [Test] + public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast([Values(1, 2, 25, 50, 99, 100, 101, 1000)] int threshold) + { + _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; + _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + _headInfo.CurrentBaseFee.Returns(0.GWei()); + int addedTxsCount = TestItem.PrivateKeys.Length; + Transaction[] transactions = new Transaction[addedTxsCount]; + + for (int i = 0; i < addedTxsCount; i++) + { + bool isBlob = i % 10 == 0; + transactions[i] = Build.A.Transaction + .WithGasPrice((addedTxsCount - i - 1).GWei()) + .WithType(isBlob ? TxType.Blob : TxType.Legacy) //some part of txs (10%) is blob type + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) + .TestObject; + + _broadcaster.Broadcast(transactions[i], true); + } + _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); + (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); + + int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); + int expectedCountOfBlobHashes = expectedCountTotal / 10 + 1; + int expectedCountOfNonBlobTxs = expectedCountTotal - expectedCountOfBlobHashes; + if (expectedCountOfNonBlobTxs > 0) + { + pickedTxs.Count.Should().Be(expectedCountOfNonBlobTxs); + } + else + { + pickedTxs.Should().BeNull(); + } + + if (expectedCountOfBlobHashes > 0) + { + pickedHashes.Count.Should().Be(expectedCountOfBlobHashes); + } + else + { + pickedHashes.Should().BeNull(); + } + List expectedTxs = new(); + List expectedHashes = new(); + for (int i = 0; i < expectedCountTotal; i++) + { + Transaction tx = transactions[i]; + + if (!tx.SupportsBlobs) + { + expectedTxs.Add(tx); + } + else + { + expectedHashes.Add(new LightTransaction(tx)); + } + } + expectedTxs.Should().BeEquivalentTo(pickedTxs); + expectedHashes.Should().BeEquivalentTo(pickedHashes); + } + + + [Test] public void should_not_pick_txs_with_GasPrice_lower_than_CurrentBaseFee([Values(1, 2, 99, 100, 101, 1000)] int threshold) { From 1d3ec8d17f1f711288b050686d5206fff2835357 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 18 Jul 2023 18:52:07 +0200 Subject: [PATCH 033/148] add test for announcing txs with MaxFeePerDataGas >= current --- .../TxBroadcasterTests.cs | 45 +++++++++++++++++-- .../Nethermind.TxPool.Test/TxPoolTests.cs | 3 +- .../Nethermind.TxPool/TxBroadcaster.cs | 6 ++- 3 files changed, 48 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 2ab602141db..78d59c77d0a 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using FluentAssertions; @@ -160,7 +161,7 @@ public void should_add_light_form_of_blob_txs_to_persistent_txs_but_not_return_i _broadcaster.Broadcast(tx, true); _broadcaster.GetSnapshot().Length.Should().Be(1); - _broadcaster.GetSnapshot()[0].Should().BeEquivalentTo(isBlob ? new LightTransaction(tx) : tx); + _broadcaster.GetSnapshot().FirstOrDefault().Should().BeEquivalentTo(isBlob ? new LightTransaction(tx) : tx); _broadcaster.TryGetPersistentTx(tx.Hash, out Transaction returnedTx).Should().Be(!isBlob); returnedTx.Should().BeEquivalentTo(isBlob ? null : tx); @@ -298,8 +299,6 @@ public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast([ expectedHashes.Should().BeEquivalentTo(pickedHashes); } - - [Test] public void should_not_pick_txs_with_GasPrice_lower_than_CurrentBaseFee([Values(1, 2, 99, 100, 101, 1000)] int threshold) { @@ -389,6 +388,46 @@ public void should_not_pick_1559_txs_with_MaxFeePerGas_lower_than_CurrentBaseFee expectedTxs.Should().BeEquivalentTo(pickedTxs, o => o.Excluding(transaction => transaction.MaxFeePerGas)); } + [Test] + public void should_not_pick_blob_txs_with_MaxFeePerDataGas_lower_than_CurrentPricePerDataGas([Values(1, 2, 99, 100, 101, 1000)] int threshold) + { + _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; + _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + + const int currentPricePerDataGasInGwei = 250; + _headInfo.CurrentPricePerDataGas.Returns(currentPricePerDataGasInGwei.GWei()); + + int addedTxsCount = TestItem.PrivateKeys.Length; + Transaction[] transactions = new Transaction[addedTxsCount]; + + for (int i = 0; i < addedTxsCount; i++) + { + transactions[i] = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerDataGas(i.GWei()) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) + .TestObject; + + _broadcaster.Broadcast(transactions[i], true); + } + + _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); + + IList pickedTxs = _broadcaster.GetPersistentTxsToSend().HashesToSend; + + int expectedCount = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount - currentPricePerDataGasInGwei); + pickedTxs.Count.Should().Be(expectedCount); + + List expectedTxs = new(); + + for (int i = 1; i <= expectedCount; i++) + { + expectedTxs.Add(transactions[addedTxsCount - i]); + } + + expectedTxs.Count(t => t.MaxFeePerDataGas >= (UInt256)currentPricePerDataGasInGwei).Should().Be(expectedCount); + } + [Test] public void should_pick_tx_with_lowest_nonce_from_bucket() { diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 06b293e433c..53dfad7af26 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -573,8 +573,7 @@ public void should_not_count_txs_with_stale_nonces_when_calculating_cumulative_c } [Test] - public void - should_add_tx_if_cost_of_executing_all_txs_in_bucket_exceeds_balance_but_these_with_lower_nonces_doesnt() + public void should_add_tx_if_cost_of_executing_all_txs_in_bucket_exceeds_balance_but_these_with_lower_nonces_doesnt() { const int gasPrice = 10; const int value = 1; diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index d07be23977f..fec050f46be 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -185,7 +185,6 @@ public void BroadcastPersistentTxs() { if (tx.MaxFeePerGas >= _headInfo.CurrentBaseFee) { - numberOfPersistentTxsToBroadcast--; if (tx.CanBeBroadcast()) { persistentTxsToSend ??= new List(numberOfPersistentTxsToBroadcast); @@ -193,9 +192,14 @@ public void BroadcastPersistentTxs() } else { + if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) // for not-blob tx MaxFeePerDataGas is null so check will be skipped + { + continue; + } persistentHashesToSend ??= new List(numberOfPersistentTxsToBroadcast); persistentHashesToSend.Add(tx); } + numberOfPersistentTxsToBroadcast--; } } else From 56d9dd44d0b5b1ca3bb0a039a0492c5b72b36a54 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 19 Jul 2023 11:06:59 +0200 Subject: [PATCH 034/148] add more tx pool metrics --- src/Nethermind/Nethermind.TxPool/Metrics.cs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 7a635ae0a13..49abd5fd053 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -83,6 +83,10 @@ public static class Metrics [Description("Number of transactions with already used nonce.")] public static long PendingTransactionsLowNonce { get; set; } + [CounterMetric] + [Description("Number of transactions rejected because of already pending tx of other type (allowed blob txs or others, not both at once).")] + public static long PendingTransactionsConflictingTxType { get; set; } + [CounterMetric] [Description("Number of pending transactions added to transaction pool.")] public static long PendingTransactionsAdded { get; set; } @@ -91,6 +95,10 @@ public static class Metrics [Description("Number of pending 1559-type transactions added to transaction pool.")] public static long Pending1559TransactionsAdded { get; set; } + [CounterMetric] + [Description("Number of pending blob-type transactions added to transaction pool.")] + public static long PendingBlobTransactionsAdded { get; set; } + [CounterMetric] [Description("Number of pending transactions evicted from transaction pool.")] public static long PendingTransactionsEvicted { get; set; } @@ -100,7 +108,7 @@ public static class Metrics public static float Eip1559TransactionsRatio { get; set; } [GaugeMetric] - [Description("Number of transactions in the block.")] + [Description("Number of blob transactions in the block.")] public static long BlobTransactionsInBlock { get; set; } [GaugeMetric] From 091bde616fc62df4e5130b9c52b8eafadfd95d21 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 19 Jul 2023 11:07:27 +0200 Subject: [PATCH 035/148] update tx pool report --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 23 ++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 7469ae14ec4..db64af3c33e 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -99,7 +99,6 @@ public TxPool(IEthereumEcdsa ecdsa, AddNodeInfoEntryForTxPool(); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - // we need some limit of blob txs, but extremely high to be limitless in practice _blobTransactions = new BlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); @@ -409,6 +408,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe (tx.SupportsBlobs ? _blobTransactions : _transactions).UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); Metrics.PendingTransactionsAdded++; if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } + if (tx.SupportsBlobs) { Metrics.PendingBlobTransactionsAdded++; } if (removed is not null) { @@ -706,6 +706,7 @@ private static void WriteTxPoolReport(ILogger logger) logger.Info(@$" Txn Pool State ({Metrics.TransactionCount:N0} txns queued) +Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) ------------------------------------------------ Sent * Transactions: {Metrics.PendingTransactionsSent,24:N0} @@ -719,12 +720,14 @@ Txn Pool State ({Metrics.TransactionCount:N0} txns queued) 3. Malformed {Metrics.PendingTransactionsMalformed,24:N0} 4. Duplicate: {Metrics.PendingTransactionsKnown,24:N0} 5. Unknown Sender: {Metrics.PendingTransactionsUnresolvableSender,24:N0} -6. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} -7. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} -8. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} -9. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} -10. Balance Too Low: {Metrics.PendingTransactionsTooLowBalance,24:N0} -11. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} +6. Conflicting TxType {Metrics.PendingTransactionsConflictingTxType,24:N0} +7. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} +8. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} +9. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} +10. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} +11. Balance Too Low: {Metrics.PendingTransactionsTooLowBalance,24:N0} +12. Failed replacement {Metrics.PendingTransactionsPassedFiltersButCannotReplace,24:N0} +13. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} ------------------------------------------------ Validated via State: {Metrics.PendingTransactionsWithExpensiveFiltering,24:N0} ------------------------------------------------ @@ -736,13 +739,17 @@ Txn Pool State ({Metrics.TransactionCount:N0} txns queued) ------------------------------------------------ Total Added: {Metrics.PendingTransactionsAdded,24:N0} * Eip1559 Added: {Metrics.Pending1559TransactionsAdded,24:N0} +* Blob Added: {Metrics.PendingBlobTransactionsAdded,24:N0} ------------------------------------------------ Total Evicted: {Metrics.PendingTransactionsEvicted,24:N0} ------------------------------------------------ -Ratios: +Ratios in last block: * Eip1559 Transactions: {Metrics.Eip1559TransactionsRatio,24:P5} * DarkPool Level1: {Metrics.DarkPoolRatioLevel1,24:P5} * DarkPool Level2: {Metrics.DarkPoolRatioLevel2,24:P5} +Amounts: +* Blob txs: {Metrics.BlobTransactionsInBlock,24:P5} +* Blobs: {Metrics.BlobsInBlock,24:P5} ------------------------------------------------ "); } From d7daff31c86e4d7be536cb7bc928e4e65b93f068 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 19 Jul 2023 11:08:05 +0200 Subject: [PATCH 036/148] add metric to filter --- .../Nethermind.TxPool/Filters/TxTypeTxFilter.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs index d0013828890..cc103fe2a25 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs @@ -22,9 +22,12 @@ public TxTypeTxFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) { - return (tx.SupportsBlobs ? _txs : _blobTxs) - .ContainsBucket(tx.SenderAddress!) // as unknownSenderFilter will run before this one - ? AcceptTxResult.PendingTxsOfOtherType - : AcceptTxResult.Accepted; + if ((tx.SupportsBlobs ? _txs : _blobTxs) + .ContainsBucket(tx.SenderAddress!)) // as unknownSenderFilter will run before this one + { + Metrics.PendingTransactionsConflictingTxType++; + return AcceptTxResult.PendingTxsOfOtherType; + } + return AcceptTxResult.Accepted; } } From fb54eb96f6387ce504c6bd868d62fb1cd9a474b2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 19 Jul 2023 11:08:24 +0200 Subject: [PATCH 037/148] improve tests --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 23 +++++++++++++------ .../Nethermind.TxPool/TxPoolConfig.cs | 2 +- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 53dfad7af26..0e5b55bed5a 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1623,7 +1623,7 @@ public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(tru .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - Transaction nonceGapBlobTx = Build.A.Transaction + Transaction nonceGapTx = Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(UInt256.One) @@ -1632,7 +1632,7 @@ public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(tru .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(nonceGapBlobTx, TxHandlingOptions.None).Should().Be(isBlob ? AcceptTxResult.NonceGap : AcceptTxResult.Accepted); + _txPool.SubmitTx(nonceGapTx, TxHandlingOptions.None).Should().Be(isBlob ? AcceptTxResult.NonceGap : AcceptTxResult.Accepted); } [Test] @@ -1738,8 +1738,9 @@ public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() [Test] public void should_add_blob_tx_and_return_when_requested() { - TxPoolConfig txPoolConfig = new TxPoolConfig() { Size = 10 }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + TxPoolConfig txPoolConfig = new() { Size = 10 }; + BlobTxStorage blobTxStorage = new(new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction blobTxAdded = Build.A.Transaction @@ -1752,7 +1753,13 @@ public void should_add_blob_tx_and_return_when_requested() _txPool.SubmitTx(blobTxAdded, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.TryGetPendingTransaction(blobTxAdded.Hash!, out Transaction blobTxReturned); - blobTxReturned.Should().BeEquivalentTo(blobTxAdded); + blobTxReturned.Should().BeEquivalentTo(blobTxAdded); // comes from cache + + blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().BeTrue(); // additional check for persistent db + blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex } [Test] @@ -1867,8 +1874,10 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(shouldReplace ? AcceptTxResult.Accepted : AcceptTxResult.ReplacementNotAllowed); _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - _txPool.TryGetPendingTransaction(shouldReplace ? secondTx.Hash! : firstTx.Hash!, out Transaction tx).Should().BeTrue(); - tx.Should().BeEquivalentTo(shouldReplace ? secondTx : firstTx); + _txPool.TryGetPendingTransaction(firstTx.Hash!, out Transaction returnedFirstTx).Should().Be(!shouldReplace); + _txPool.TryGetPendingTransaction(secondTx.Hash!, out Transaction returnedSecondTx).Should().Be(shouldReplace); + returnedFirstTx.Should().BeEquivalentTo(shouldReplace ? null : firstTx); + returnedSecondTx.Should().BeEquivalentTo(shouldReplace ? secondTx : null); } [TestCase(0, 97)] diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 26e55136ce7..bde79d2bbe7 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,7 +7,7 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; - public int BlobPoolSize { get; set; } = 128 * 1024; + public int BlobPoolSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice public int BlobCacheSize { get; set; } = 256; public int HashCacheSize { get; set; } = 512 * 1024; public long? GasLimit { get; set; } = null; From d8cf26173e24357a6c3ad0b451bdb5be847a8d64 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 19 Jul 2023 17:29:28 +0200 Subject: [PATCH 038/148] refactors --- .../Nethermind.TxPool/BlobTxStorage.cs | 6 +++-- .../Collections/BlobTxDistinctSortedPool.cs | 22 +++++++++++++++++-- .../Nethermind.TxPool/TxBroadcaster.cs | 4 ++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 +--- 4 files changed, 27 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 145d3ca9c82..2cd63135736 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -19,7 +19,8 @@ public BlobTxStorage(IDb database) _database = database ?? throw new ArgumentNullException(nameof(database)); } - public bool TryGet(Keccak hash, out Transaction? transaction) => TryDecode(_database.Get(hash), out transaction); + public bool TryGet(Keccak hash, out Transaction? transaction) + => TryDecode(_database.Get(hash), out transaction); private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) { @@ -61,5 +62,6 @@ public void Add(Transaction transaction) _database.Set(transaction.Hash, Rlp.Encode(transaction, RlpBehaviors.InMempoolForm).Bytes); } - public void Delete(ValueKeccak hash) => _database.Remove(hash.Bytes); + public void Delete(ValueKeccak hash) + => _database.Remove(hash.Bytes); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 7027418c0ad..c15f437f3bc 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -26,7 +26,8 @@ public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfig txPoolCo RecreateLightTxCollectionAndCache(blobTxStorage); } - protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); + protected override IComparer GetReplacementComparer(IComparer comparer) + => comparer.GetBlobReplacementComparer(); private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { @@ -82,10 +83,27 @@ public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) return false; } - // only for tests - to test sorting + /// + /// For tests only - to test sorting + /// internal bool TryGetLightValue(Keccak hash, out Transaction? lightBlobTx) => base.TryGetValue(hash, out lightBlobTx); + /// + /// Removing blob tx from in-memory collections and still keeps in persistent db in case of reorgs + /// + public bool TryRemoveProcessedButNotFinalizedBlobTx(ValueKeccak hash, out Transaction? removed) + { + if (base.TryRemove(hash, out removed)) + { + // save somewhere to delete after finalization + + return true; + } + + return false; + } + protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index fec050f46be..ea9c03216f2 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -192,8 +192,8 @@ public void BroadcastPersistentTxs() } else { - if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) // for not-blob tx MaxFeePerDataGas is null so check will be skipped - { + if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) // for not-blob tx MaxFeePerDataGas + { // is null so check will be skipped continue; } persistentHashesToSend ??= new List(numberOfPersistentTxsToBroadcast); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index db64af3c33e..41db363d3a1 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -574,9 +574,7 @@ public bool RemoveTransaction(Keccak? hash) hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction); if (!hasBeenRemoved) { - // add flag keepInDb if coming from processed block? and remove after finalization? - // actually for blobs it is removed here only after block processing - hasBeenRemoved = _blobTransactions.TryRemove(hash, out transaction); + hasBeenRemoved = _blobTransactions.TryRemoveProcessedButNotFinalizedBlobTx(hash, out transaction); } if (transaction is null || !hasBeenRemoved) From 647873fad37303bf1225eb279e9f708260375801 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 14:30:31 +0200 Subject: [PATCH 039/148] clean --- .../Collections/BlobTxDistinctSortedPool.cs | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index c15f437f3bc..c56cdabd229 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -89,21 +89,6 @@ public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) internal bool TryGetLightValue(Keccak hash, out Transaction? lightBlobTx) => base.TryGetValue(hash, out lightBlobTx); - /// - /// Removing blob tx from in-memory collections and still keeps in persistent db in case of reorgs - /// - public bool TryRemoveProcessedButNotFinalizedBlobTx(ValueKeccak hash, out Transaction? removed) - { - if (base.TryRemove(hash, out removed)) - { - // save somewhere to delete after finalization - - return true; - } - - return false; - } - protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); From 5fdff5a751308af006ec549063fd1e88c80e4efe Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 14:30:48 +0200 Subject: [PATCH 040/148] add test --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 0e5b55bed5a..37d9a22f8dd 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1762,6 +1762,20 @@ public void should_add_blob_tx_and_return_when_requested() .Excluding(t => t.PoolIndex)); // ...and PoolIndex } + [Test] + public void should_not_throw_when_asking_for_non_existing_tx() + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + BlobTxStorage blobTxStorage = new(new MemDb(), new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + + _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); + blobTxReturned.Should().BeNull(); + + blobTxStorage.TryGet(TestItem.KeccakA, out Transaction blobTxFromDb).Should().BeFalse(); + blobTxFromDb.Should().BeNull(); + } + [Test] public void should_remove_replaced_blob_tx() { From 12e9b9acd5d8314fd6bc5ac0fd1d981f9cb7382c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 14:32:45 +0200 Subject: [PATCH 041/148] fix TxPool --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 41db363d3a1..c0c7d155352 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -574,7 +574,7 @@ public bool RemoveTransaction(Keccak? hash) hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction); if (!hasBeenRemoved) { - hasBeenRemoved = _blobTransactions.TryRemoveProcessedButNotFinalizedBlobTx(hash, out transaction); + hasBeenRemoved = _blobTransactions.TryRemove(hash, out transaction); } if (transaction is null || !hasBeenRemoved) From 53f44156e4be740a7772482e4e6fa97af55e70c8 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 20 Jul 2023 15:40:06 +0200 Subject: [PATCH 042/148] fix test --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 37d9a22f8dd..b00aa3230a4 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1766,7 +1766,7 @@ public void should_add_blob_tx_and_return_when_requested() public void should_not_throw_when_asking_for_non_existing_tx() { TxPoolConfig txPoolConfig = new() { Size = 10 }; - BlobTxStorage blobTxStorage = new(new MemDb(), new MemDb()); + BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); From e6469403df92cb5107444db0bdd1a5268614b14c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 21 Jul 2023 13:09:27 +0200 Subject: [PATCH 043/148] fix EthereumTests --- src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index 8e3e3a147db..f8c4edc4a4f 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -133,7 +133,14 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); IStateReader stateReader = new StateReader(trieStore, codeDb, _logManager); IChainHeadInfoProvider chainHeadInfoProvider = new ChainHeadInfoProvider(specProvider, blockTree, stateReader); - ITxPool transactionPool = new TxPool(ecdsa, chainHeadInfoProvider, new TxPoolConfig(), new TxValidator(specProvider.ChainId), _logManager, transactionComparerProvider.GetDefaultComparer()); + ITxStorage txStorage = new BlobTxStorage(new MemDb()); + ITxPool transactionPool = new TxPool(ecdsa, + txStorage, + chainHeadInfoProvider, + new TxPoolConfig(), + new TxValidator(specProvider.ChainId), + _logManager, + transactionComparerProvider.GetDefaultComparer()); IReceiptStorage receiptStorage = NullReceiptStorage.Instance; IBlockhashProvider blockhashProvider = new BlockhashProvider(blockTree, _logManager); From dbd8890ce7758bbb26b825c4bc4d7ba64916b71d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 21 Jul 2023 13:09:38 +0200 Subject: [PATCH 044/148] fix report --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index c0c7d155352..f68959ec3ee 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -746,8 +746,8 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) * DarkPool Level1: {Metrics.DarkPoolRatioLevel1,24:P5} * DarkPool Level2: {Metrics.DarkPoolRatioLevel2,24:P5} Amounts: -* Blob txs: {Metrics.BlobTransactionsInBlock,24:P5} -* Blobs: {Metrics.BlobsInBlock,24:P5} +* Blob txs: {Metrics.BlobTransactionsInBlock,24:N0} +* Blobs: {Metrics.BlobsInBlock,24:N0} ------------------------------------------------ "); } From 82a51106feb6bee4e14573db0202d017c0f23257 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 21 Jul 2023 13:12:05 +0200 Subject: [PATCH 045/148] fix whitespaces --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index b00aa3230a4..946f23b1200 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1612,7 +1612,7 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g [Test] public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) { - _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction @@ -1649,7 +1649,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); @@ -1673,7 +1673,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); @@ -1816,7 +1816,7 @@ public void should_remove_replaced_blob_tx() _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); blobTxReturned.Should().BeEquivalentTo(newTx); blobTxStorage.TryGet(oldTx.Hash, out blobTxFromDb).Should().BeFalse(); - blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); + blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); blobTxFromDb.Should().BeEquivalentTo(newTx, options => options .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... @@ -1864,7 +1864,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; const int initialFee = 10; - _txPool = CreatePool(new TxPoolConfig(){ Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction From 44b145e590bb142e0a39060d981a73a278e8fd0b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 25 Jul 2023 14:35:55 +0200 Subject: [PATCH 046/148] fix blob txs picking when building block --- .../Producers/TxPoolTxSource.cs | 24 +++++++++++- .../Collections/BlobTxDistinctSortedPool.cs | 38 +++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index f0afa9eb19b..ba45eb5bf6f 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -60,6 +60,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi int i = 0; int blobsCounter = 0; UInt256 dataGasPrice = UInt256.Zero; + List? selectedBlobTxs = null; foreach (Transaction blobTx in blobTransactions) { @@ -110,7 +111,8 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {blobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); selectedTransactions++; - yield return blobTx; + selectedBlobTxs ??= new List((int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob)); + selectedBlobTxs.Add(blobTx); } foreach (Transaction tx in transactions) @@ -129,10 +131,30 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); + if (selectedBlobTxs?.Count > 0) + { + foreach (Transaction blobTx in new List(selectedBlobTxs)) + { + if (comparer.Compare(blobTx, tx) > 0) + { + yield return blobTx; + selectedBlobTxs.Remove(blobTx); + } + } + } + selectedTransactions++; yield return tx; } + if (selectedBlobTxs?.Count > 0) + { + foreach (Transaction blobTx in selectedBlobTxs) + { + yield return blobTx; + } + } + if (_logger.IsDebug) _logger.Debug($"Potentially selected {selectedTransactions} out of {i} pending transactions checked."); } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index c56cdabd229..4229497226d 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -12,6 +12,7 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { + const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob); private readonly ITxStorage _blobTxStorage; private readonly LruCache _blobTxCache; private readonly ILogger _logger; @@ -53,14 +54,43 @@ public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) return false; } + // ToDo: add synchronized? public IEnumerable GetBlobTransactions() { - // ToDo: to refactor - it must lazy enumerate starting from the best - foreach (Transaction lightBlobTx in GetSnapshot()) + int pickedBlobs = 0; + List? blobTxsToReadd = null; + + while (pickedBlobs < MaxNumberOfBlobsInBlock) { - TryGetValue(lightBlobTx.Hash!, out Transaction? fullBlobTx); - yield return fullBlobTx!; + Transaction? bestTxLight = GetFirsts().Min; + + if (bestTxLight?.Hash is null || bestTxLight.BlobVersionedHashes is null) + { + break; + } + + if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + bestTxLight.BlobVersionedHashes.Length <= MaxNumberOfBlobsInBlock) + { + yield return fullBlobTx!; + pickedBlobs++; + } + + if (TryRemove(bestTxLight.Hash)) + { + blobTxsToReadd ??= new(MaxNumberOfBlobsInBlock); + blobTxsToReadd.Add(fullBlobTx!); + } + } + + + if (blobTxsToReadd is not null) + { + foreach (Transaction fullBlobTx in blobTxsToReadd) + { + TryInsert(fullBlobTx!, out Transaction? removed); + } } + } public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) From 5a6cf30e2c4f0bb20596f423b2b383fc1e78cb4c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 25 Jul 2023 14:36:02 +0200 Subject: [PATCH 047/148] fix test --- .../Nethermind.Blockchain.Test/TransactionSelectorTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index f9d8de672ec..5efd1ed3615 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -241,6 +241,7 @@ void SetAccountStates(IEnumerable
missingAddresses) Transaction[] blobTransactions = testCase.Transactions .Where(t => t?.SenderAddress is not null) .Where(t => t.SupportsBlobs) + .OrderBy(t => t, comparer) .ToArray(); transactionPool.GetPendingTransactionsBySender().Returns(transactions); transactionPool.GetPendingBlobTransactions().Returns(blobTransactions); From 771858c26eb8fa1e2707b42dab0f0aee30cbc744 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 25 Jul 2023 15:06:14 +0200 Subject: [PATCH 048/148] fix benchmarks solution --- .../Eth62ProtocolHandlerBenchmarks.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs b/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs index 34616fb7701..713a268c7e7 100644 --- a/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs +++ b/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs @@ -57,6 +57,7 @@ public void SetUp() var specProvider = MainnetSpecProvider.Instance; TxPool.TxPool txPool = new TxPool.TxPool( ecdsa, + new BlobTxStorage(new MemDb()), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(MainnetSpecProvider.Instance), tree, stateProvider), new TxPoolConfig(), new TxValidator(TestBlockchainIds.ChainId), From f4df11210e545beb61f4b69a15e6834a4c1e0df9 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 26 Jul 2023 11:55:57 +0200 Subject: [PATCH 049/148] add filtering for MaxPriorityFeePerGas < 1GWei for blob transactions --- .../Nethermind.TxPool/Filters/FeeTooLowFilter.cs | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs index 577f3324507..6c15155b11b 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs @@ -3,6 +3,7 @@ using System.Diagnostics; using Nethermind.Core; +using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Logging; @@ -32,6 +33,13 @@ public FeeTooLowFilter(IChainHeadInfoProvider headInfo, TxDistinctSortedPool txs public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { + if (tx.SupportsBlobs && tx.MaxPriorityFeePerGas < 1.GWei()) + { + Metrics.PendingTransactionsTooLowFee++; + if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, too low payable gas price with options {handlingOptions} from {new StackTrace()}"); + return AcceptTxResult.FeeTooLow.WithMessage($"MaxPriorityFeePerGas for blob transaction needs to be at least {1.GWei()} (1 GWei), is {tx.MaxPriorityFeePerGas}."); + } + bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; if (isLocal || tx.SupportsBlobs) { From 965a003230540b126b14d0de6cff9f5a907a4a9e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 26 Jul 2023 12:12:09 +0200 Subject: [PATCH 050/148] adjust tests to min requirement of 1 gwei --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 52 +++++++++---------- 1 file changed, 25 insertions(+), 27 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 946f23b1200..d7e5489a9ba 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1618,16 +1618,16 @@ public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(tru Transaction firstTx = Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; Transaction nonceGapTx = Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce((UInt256)2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; @@ -1643,8 +1643,8 @@ Transaction GetTx(bool isBlob, UInt256 nonce) return Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } @@ -1667,8 +1667,8 @@ Transaction GetTx(bool isBlob, UInt256 nonce) return Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } @@ -1708,8 +1708,8 @@ public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() Transaction tx = Build.A.Transaction .WithNonce((UInt256)i) .WithType(TxType.EIP1559) - .WithMaxFeePerGas((UInt256)(100 - i)) - .WithMaxPriorityFeePerGas((UInt256)(100 - i)) + .WithMaxFeePerGas(1.GWei() + (UInt256)(100 - i)) + .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)(100 - i)) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); } @@ -1718,15 +1718,15 @@ public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() Transaction feeTooLow1559Tx = Build.A.Transaction .WithType(TxType.EIP1559) - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei() + UInt256.One) + .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; Transaction feeTooLowBlobTx = Build.A.Transaction .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei() + UInt256.One) + .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; @@ -1745,8 +1745,8 @@ public void should_add_blob_tx_and_return_when_requested() Transaction blobTxAdded = Build.A.Transaction .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; @@ -1779,7 +1779,6 @@ public void should_not_throw_when_asking_for_non_existing_tx() [Test] public void should_remove_replaced_blob_tx() { - const int initialFee = 10; TxPoolConfig txPoolConfig = new() { Size = 10 }; BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); @@ -1788,8 +1787,8 @@ public void should_remove_replaced_blob_tx() Transaction oldTx = Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(initialFee) - .WithMaxPriorityFeePerGas(initialFee) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; Transaction newTx = Build.A.Transaction @@ -1836,8 +1835,8 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ .WithType(isBlob ? TxType.Blob : TxType.EIP1559) .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(UInt256.One) - .WithMaxPriorityFeePerGas(UInt256.One) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .WithMaxFeePerDataGas(isBlob ? UInt256.One : null) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; @@ -1852,9 +1851,9 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ } else { - _txPool.TryGetPendingTransaction(tx.Hash!, out Transaction eip1559tx); - eip1559tx.Should().BeEquivalentTo(tx); - eip1559tx.GasBottleneck.Should().Be(UInt256.One); + _txPool.TryGetPendingTransaction(tx.Hash!, out Transaction eip1559Tx); + eip1559Tx.Should().BeEquivalentTo(tx); + eip1559Tx.GasBottleneck.Should().Be(1.GWei()); } } @@ -1862,7 +1861,6 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, 2, 3, 4, 5, 6)] int blobsInFirstTx, [Values(1, 2, 3, 4, 5, 6)] int blobsInSecondTx) { bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; - const int initialFee = 10; _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1870,8 +1868,8 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, Transaction firstTx = Build.A.Transaction .WithShardBlobTxTypeAndFields(blobsInFirstTx) .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(initialFee) - .WithMaxPriorityFeePerGas(initialFee) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; Transaction secondTx = Build.A.Transaction From 6f045a60c00907fd1796fadb7e31a9563bdedc6f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 26 Jul 2023 12:20:20 +0200 Subject: [PATCH 051/148] add test --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index d7e5489a9ba..cdcf5f88595 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1609,6 +1609,28 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); } + [TestCase(999_999_999, false)] + [TestCase(1_000_000_000, true)] + public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than_1GWei(int maxPriorityFeePerGas, bool expectedResult) + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + _headInfo.CurrentPricePerDataGas = UInt256.MaxValue; + + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas((UInt256)maxPriorityFeePerGas) + .WithMaxPriorityFeePerGas((UInt256)maxPriorityFeePerGas) + .WithMaxFeePerDataGas(UInt256.One) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(expectedResult + ? AcceptTxResult.Accepted + : AcceptTxResult.FeeTooLow); + } + [Test] public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) { From c3fbde1031e1ea12ee5d08ff26e7ac2f8802ef04 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 13:07:18 +0200 Subject: [PATCH 052/148] add more txpool config options --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 10 ++++++++-- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 4 +++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 289fce44f8e..a7c3a1d8091 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -13,12 +13,18 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "Max number of transactions held in mempool (more transactions in mempool mean more memory used")] int Size { get; set; } + [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] + bool PersistentBlobPoolEnabled { get; set; } + [ConfigItem(DefaultValue = "131072", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] - int BlobPoolSize { get; set; } + int PersistentBlobPoolSize { get; set; } - [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory (recent ones)")] + [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage")] int BlobCacheSize { get; set; } + [ConfigItem(DefaultValue = "512", Description = "Max number of full blob transactions stored in memory. Used only if persistent storage is disabled")] + int InMemoryBlobPoolSize { get; set; } + [ConfigItem(DefaultValue = "524288", Description = "Max number of cached hashes of already known transactions." + "It is set automatically by the memory hint.")] diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index bde79d2bbe7..9b5efe627e7 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,8 +7,10 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; - public int BlobPoolSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice + public bool PersistentBlobPoolEnabled { get; set; } = false; + public int PersistentBlobPoolSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice public int BlobCacheSize { get; set; } = 256; + public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled public int HashCacheSize { get; set; } = 512 * 1024; public long? GasLimit { get; set; } = null; public int? ReportMinutes { get; set; } = null; From 4aa0e0eeda0633051c2f21f23bd3de1711afb622 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 13:15:55 +0200 Subject: [PATCH 053/148] adjust storage --- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 5 +++-- src/Nethermind/Nethermind.TxPool/ITxStorage.cs | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 2cd63135736..2896aa6b5a2 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Db; @@ -19,8 +20,8 @@ public BlobTxStorage(IDb database) _database = database ?? throw new ArgumentNullException(nameof(database)); } - public bool TryGet(Keccak hash, out Transaction? transaction) - => TryDecode(_database.Get(hash), out transaction); + public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) + => TryDecode(_database.Get(hash.Bytes), out transaction); private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) { diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 1fb6b206b1f..380dfe53245 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -9,7 +10,7 @@ namespace Nethermind.TxPool; public interface ITxStorage { - bool TryGet(Keccak hash, out Transaction? transaction); + bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction); IEnumerable GetAll(); void Add(Transaction transaction); void Delete(ValueKeccak hash); From a91b8d8b3e9f815f0f8477399db2e58d4e6287bb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 13:40:47 +0200 Subject: [PATCH 054/148] add non-persistent blob tx collection --- .../Producers/TxPoolTxSource.cs | 2 - .../Collections/BlobTxDistinctSortedPool.cs | 94 ++++--------------- .../Collections/SortedPool.cs | 11 +++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 20 ++-- 4 files changed, 40 insertions(+), 87 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index ba45eb5bf6f..4ed3ef6de3d 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -72,8 +72,6 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi i++; - //add removal of null sender txs? There shouldn't be null sender at this point - bool success = _txFilterPipeline.Execute(blobTx, parent); if (!success) continue; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 4229497226d..2aece93653f 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -1,10 +1,8 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Collections.Generic; using Nethermind.Core; -using Nethermind.Core.Caching; using Nethermind.Core.Crypto; using Nethermind.Logging; @@ -12,117 +10,61 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { - const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob); - private readonly ITxStorage _blobTxStorage; - private readonly LruCache _blobTxCache; - private readonly ILogger _logger; + protected const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob); - public BlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfig txPoolConfig, IComparer comparer, ILogManager logManager) - : base(txPoolConfig.BlobPoolSize, comparer, logManager) + public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) + : base(capacity, comparer, logManager) { - _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); - _blobTxCache = new(txPoolConfig.BlobCacheSize, txPoolConfig.BlobCacheSize, "blob txs cache"); - _logger = logManager.GetClassLogger(); - - RecreateLightTxCollectionAndCache(blobTxStorage); } protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) - { - if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); - foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) - { - if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx))) - { - _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); - } - } - } - - public bool TryInsert(Transaction fullBlobTx, out Transaction? removed) - { - if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out removed)) - { - _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); - _blobTxStorage.Add(fullBlobTx); - return true; - } + public virtual bool TryInsertBlobTx(Transaction fullBlobTx, out Transaction? removed) + => base.TryInsert(fullBlobTx.Hash, fullBlobTx, out removed); - return false; - } + public virtual bool TryGetBlobTx(ValueKeccak hash, out Transaction? fullBlobTx) + => base.TryGetValue(hash, out fullBlobTx); - // ToDo: add synchronized? - public IEnumerable GetBlobTransactions() + public virtual IEnumerable GetBlobTransactions() { int pickedBlobs = 0; List? blobTxsToReadd = null; while (pickedBlobs < MaxNumberOfBlobsInBlock) { - Transaction? bestTxLight = GetFirsts().Min; + Transaction? bestTx = GetFirsts().Min; - if (bestTxLight?.Hash is null || bestTxLight.BlobVersionedHashes is null) + if (bestTx?.Hash is null || bestTx.BlobVersionedHashes is null) { break; } - if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + bestTxLight.BlobVersionedHashes.Length <= MaxNumberOfBlobsInBlock) + if (pickedBlobs + bestTx.BlobVersionedHashes.Length <= MaxNumberOfBlobsInBlock) { - yield return fullBlobTx!; - pickedBlobs++; + yield return bestTx; + pickedBlobs += bestTx.BlobVersionedHashes.Length; } - if (TryRemove(bestTxLight.Hash)) + if (TryRemove(bestTx.Hash)) { blobTxsToReadd ??= new(MaxNumberOfBlobsInBlock); - blobTxsToReadd.Add(fullBlobTx!); + blobTxsToReadd.Add(bestTx!); } } - if (blobTxsToReadd is not null) { - foreach (Transaction fullBlobTx in blobTxsToReadd) - { - TryInsert(fullBlobTx!, out Transaction? removed); - } - } - - } - - public bool TryGetValue(Keccak hash, out Transaction? fullBlobTx) - { - if (base.TryGetValue(hash, out Transaction? lightBlobTx)) - { - if (_blobTxCache.TryGet(hash, out fullBlobTx)) + foreach (Transaction blobTx in blobTxsToReadd) { - return true; - } - - if (_blobTxStorage.TryGet(hash, out fullBlobTx)) - { - _blobTxCache.Set(hash, fullBlobTx!); - return true; + TryInsert(blobTx.Hash, blobTx!, out Transaction? removed); } } - - fullBlobTx = default; - return false; } /// /// For tests only - to test sorting /// - internal bool TryGetLightValue(Keccak hash, out Transaction? lightBlobTx) + internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? lightBlobTx) => base.TryGetValue(hash, out lightBlobTx); - - protected override bool Remove(ValueKeccak hash, Transaction tx) - { - _blobTxCache.Delete(hash); - _blobTxStorage.Delete(hash); - return base.Remove(hash, tx); - } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 024ecf719a4..5acd3b00679 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -255,6 +255,17 @@ public IEnumerable TakeWhile(TGroupKey groupKey, Predicate where return Enumerable.Empty(); } + /// + /// Checks if element is present. + /// + /// Key to check presence. + /// True if element is present in pool. + [MethodImpl(MethodImplOptions.Synchronized)] + protected bool ContainsValue(TKey key) + { + return _cacheMap.ContainsKey(key); + } + /// /// Tries to get element. /// diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index f68959ec3ee..02bb2386e7d 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -99,7 +99,9 @@ public TxPool(IEthereumEcdsa ecdsa, AddNodeInfoEntryForTxPool(); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - _blobTransactions = new BlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager); + _blobTransactions = txPoolConfig.PersistentBlobPoolEnabled + ? new PersistentBlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager) + : new BlobTxDistinctSortedPool(_txPoolConfig.InMemoryBlobPoolSize, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _headInfo.HeadChanged += OnHeadChange; @@ -385,7 +387,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe : worstTx.GasBottleneck; bool inserted = (tx.SupportsBlobs - ? _blobTransactions.TryInsert(tx, out Transaction? removed) + ? _blobTransactions.TryInsertBlobTx(tx, out Transaction? removed) : _transactions.TryInsert(tx.Hash!, tx, out removed)); if (!inserted) @@ -498,11 +500,11 @@ private void UpdateBuckets() _transactions.UpdatePool(_accounts, _updateBucket); // ensure the capacity of the blob pool - if (_blobTransactions.Count > _txPoolConfig.BlobPoolSize) - if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.BlobPoolSize}"); + if (_blobTransactions.Count > _txPoolConfig.PersistentBlobPoolSize) + if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.PersistentBlobPoolSize}"); - if (_blobTransactions.Count == _txPoolConfig.BlobPoolSize) - if (_logger.IsDebug) _logger.Debug($"Blob TxPool has reached max size of {_txPoolConfig.BlobPoolSize}, blob txs can be evicted now"); + if (_blobTransactions.Count == _txPoolConfig.PersistentBlobPoolSize) + if (_logger.IsDebug) _logger.Debug($"Blob TxPool has reached max size of {_txPoolConfig.PersistentBlobPoolSize}, blob txs can be evicted now"); _blobTransactions.UpdatePool(_accounts, _updateBucket); } @@ -597,14 +599,14 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) lock (_locker) { return _transactions.TryGetValue(hash, out transaction) - || _blobTransactions.TryGetValue(hash, out transaction) + || _blobTransactions.TryGetBlobTx(hash, out transaction) || _broadcaster.TryGetPersistentTx(hash, out transaction); } } // only for tests - to test sorting - internal bool TryGetLightBlobTransaction(Keccak hash, out Transaction? transaction) - => _blobTransactions.TryGetLightValue(hash, out transaction); + internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? transaction) + => _blobTransactions.TryGetBlobTxSortingEquivalent(hash, out transaction); // should own transactions (in broadcaster) be also checked here? // maybe it should use NonceManager, as it already has info about local txs? From 0ea2bc566ce3c89c9d22ae8f8ff29d3b1d799d01 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 13:41:15 +0200 Subject: [PATCH 055/148] one more --- .../PersistentBlobTxDistinctSortedPool.cs | 118 ++++++++++++++++++ 1 file changed, 118 insertions(+) create mode 100644 src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs new file mode 100644 index 00000000000..143e3cd636e --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -0,0 +1,118 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using Nethermind.Core; +using Nethermind.Core.Caching; +using Nethermind.Core.Crypto; +using Nethermind.Logging; + +namespace Nethermind.TxPool.Collections; + +public class PersistentBlobTxDistinctSortedPool : BlobTxDistinctSortedPool +{ + private readonly ITxStorage _blobTxStorage; + private readonly LruCache _blobTxCache; + private readonly ILogger _logger; + + public PersistentBlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfig txPoolConfig, IComparer comparer, ILogManager logManager) + : base(txPoolConfig.PersistentBlobPoolSize, comparer, logManager) + { + _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); + _blobTxCache = new(txPoolConfig.BlobCacheSize, txPoolConfig.BlobCacheSize, "blob txs cache"); + _logger = logManager.GetClassLogger(); + + RecreateLightTxCollectionAndCache(blobTxStorage); + } + + private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) + { + if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); + foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) + { + if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx))) + { + _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); + } + } + } + + public override bool TryInsertBlobTx(Transaction fullBlobTx, out Transaction? removed) + { + if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out removed)) + { + _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); + _blobTxStorage.Add(fullBlobTx); + return true; + } + + return false; + } + + public override bool TryGetBlobTx(ValueKeccak hash, out Transaction? fullBlobTx) + { + if (base.ContainsValue(hash)) + { + if (_blobTxCache.TryGet(hash, out fullBlobTx)) + { + return true; + } + + if (_blobTxStorage.TryGet(hash, out fullBlobTx)) + { + _blobTxCache.Set(hash, fullBlobTx); + return true; + } + } + + fullBlobTx = default; + return false; + } + + // ToDo: add synchronized? + public override IEnumerable GetBlobTransactions() + { + int pickedBlobs = 0; + List? blobTxsToReadd = null; + + while (pickedBlobs < MaxNumberOfBlobsInBlock) + { + Transaction? bestTxLight = GetFirsts().Min; + + if (bestTxLight?.Hash is null) + { + break; + } + + if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + fullBlobTx.BlobVersionedHashes!.Length <= MaxNumberOfBlobsInBlock) + { + yield return fullBlobTx!; + pickedBlobs += fullBlobTx.BlobVersionedHashes.Length; + } + + if (TryRemove(bestTxLight.Hash)) + { + blobTxsToReadd ??= new(MaxNumberOfBlobsInBlock); + blobTxsToReadd.Add(fullBlobTx!); + } + } + + + if (blobTxsToReadd is not null) + { + foreach (Transaction fullBlobTx in blobTxsToReadd) + { + TryInsert(fullBlobTx.Hash, fullBlobTx!, out Transaction? removed); + } + } + + } + + protected override bool Remove(ValueKeccak hash, Transaction tx) + { + _blobTxCache.Delete(hash); + _blobTxStorage.Delete(hash); + return base.Remove(hash, tx); + } +} From 17e997a7bb042bd3e16b8f6f2ed00b958f555f8d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 13:42:02 +0200 Subject: [PATCH 056/148] adjust tests, add new one --- .../Nethermind.TxPool.Test/TxPoolTests.cs | 56 ++++++++++++++----- 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index cdcf5f88595..530a0ea4568 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1758,9 +1758,9 @@ public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() } [Test] - public void should_add_blob_tx_and_return_when_requested() + public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1775,13 +1775,16 @@ public void should_add_blob_tx_and_return_when_requested() _txPool.SubmitTx(blobTxAdded, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.TryGetPendingTransaction(blobTxAdded.Hash!, out Transaction blobTxReturned); - blobTxReturned.Should().BeEquivalentTo(blobTxAdded); // comes from cache + blobTxReturned.Should().BeEquivalentTo(blobTxAdded); - blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().BeTrue(); // additional check for persistent db - blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex + blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().Be(isPersistentStorage); // additional check for persistent db + if (isPersistentStorage) + { + blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex + } } [Test] @@ -1801,7 +1804,7 @@ public void should_not_throw_when_asking_for_non_existing_tx() [Test] public void should_remove_replaced_blob_tx() { - TxPoolConfig txPoolConfig = new() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = true }; BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1845,9 +1848,32 @@ public void should_remove_replaced_blob_tx() } [Test] - public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_lower_than_current([Values(true, false)] bool isBlob) + public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_storage_enabled([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerDataGas(UInt256.One) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(0); + + _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); + returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx); + } + + [Test] + public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) + { + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1867,9 +1893,11 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ _txPool.GetPendingTransactionsCount().Should().Be(isBlob ? 0 : 1); if (isBlob) { - _txPool.TryGetLightBlobTransaction(tx.Hash!, out Transaction blobTxReturned); - blobTxReturned.Should().NotBeEquivalentTo(tx); - blobTxReturned.GasBottleneck.Should().Be(UInt256.Zero); + _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); + returned.GasBottleneck.Should().Be(UInt256.Zero); + returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx, + options => options.Excluding(t => t.GasBottleneck)); + returned.Should().NotBeEquivalentTo(isPersistentStorage ? tx : new LightTransaction(tx)); } else { From 7c71180eaa05d39a8244d742f3e7f9ccfa87f7b0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 15:15:48 +0200 Subject: [PATCH 057/148] post merge fixes --- .../ChainHeadInfoProvider.cs | 6 +++--- .../Producers/TxPoolTxSource.cs | 16 ++++++++-------- .../TxBroadcasterTests.cs | 12 ++++++------ .../Nethermind.TxPool.Test/TxPoolTests.cs | 18 +++++++++--------- .../Collections/BlobTxDistinctSortedPool.cs | 2 +- .../Comparison/CompareReplacedBlobTx.cs | 2 +- .../IChainHeadInfoProvider.cs | 2 +- .../Nethermind.TxPool/LightTransaction.cs | 2 +- .../Nethermind.TxPool/TransactionExtensions.cs | 4 ++-- .../Nethermind.TxPool/TxBroadcaster.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 11 files changed, 34 insertions(+), 34 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs index 03cfefdfdda..b3400e2ab5f 100644 --- a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs +++ b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs @@ -43,7 +43,7 @@ public ChainHeadInfoProvider(IChainHeadSpecProvider specProvider, IBlockTree blo public UInt256 CurrentBaseFee { get; private set; } - public UInt256 CurrentPricePerDataGas { get; internal set; } + public UInt256 CurrentPricePerBlobGas { get; internal set; } public event EventHandler? HeadChanged; @@ -51,8 +51,8 @@ private void OnHeadChanged(object? sender, BlockReplacementEventArgs e) { BlockGasLimit = e.Block!.GasLimit; CurrentBaseFee = e.Block.Header.BaseFeePerGas; - CurrentPricePerDataGas = - DataGasCalculator.TryCalculateDataGasPricePerUnit(e.Block.Header, out UInt256 currentPricePerDataGas) + CurrentPricePerBlobGas = + BlobGasCalculator.TryCalculateBlobGasPricePerUnit(e.Block.Header, out UInt256 currentPricePerDataGas) ? currentPricePerDataGas : UInt256.Zero; HeadChanged?.Invoke(sender, e); diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 33704504f03..90de3869999 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -64,7 +64,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi foreach (Transaction blobTx in blobTransactions) { - if (DataGasCalculator.CalculateDataGas(blobsCounter) == Eip4844Constants.MaxDataGasPerBlock) + if (BlobGasCalculator.CalculateBlobGas(blobsCounter) == Eip4844Constants.MaxBlobGasPerBlock) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); break; @@ -75,15 +75,15 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi bool success = _txFilterPipeline.Execute(blobTx, parent); if (!success) continue; - if (dataGasPrice.IsZero) + if (blobGasPrice.IsZero) { - ulong? excessDataGas = DataGasCalculator.CalculateExcessDataGas(parent, _specProvider.GetSpec(parent)); + ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, _specProvider.GetSpec(parent)); if (excessDataGas is null) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); continue; } - if (!DataGasCalculator.TryCalculateDataGasPricePerUnit(excessDataGas.Value, out dataGasPrice)) + if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(excessDataGas.Value, out blobGasPrice)) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to calculate data gas price."); continue; @@ -92,14 +92,14 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi int txAmountOfBlobs = blobTx.BlobVersionedHashes?.Length ?? 0; - if (dataGasPrice > blobTx.MaxFeePerDataGas) + if (blobGasPrice > blobTx.MaxFeePerBlobGas) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, data gas fee is too low."); continue; } - if (DataGasCalculator.CalculateDataGas(blobsCounter + txAmountOfBlobs) > - Eip4844Constants.MaxDataGasPerBlock) + if (BlobGasCalculator.CalculateBlobGas(blobsCounter + txAmountOfBlobs) > + Eip4844Constants.MaxBlobGasPerBlock) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, not enough blob space."); continue; @@ -109,7 +109,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {blobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); selectedTransactions++; - selectedBlobTxs ??= new List((int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob)); + selectedBlobTxs ??= new List((int)(Eip4844Constants.MaxBlobGasPerBlock / Eip4844Constants.BlobGasPerBlob)); selectedBlobTxs.Add(blobTx); } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index b4afde20fea..7f855aa8f1b 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -389,13 +389,13 @@ public void should_not_pick_1559_txs_with_MaxFeePerGas_lower_than_CurrentBaseFee } [Test] - public void should_not_pick_blob_txs_with_MaxFeePerDataGas_lower_than_CurrentPricePerDataGas([Values(1, 2, 99, 100, 101, 1000)] int threshold) + public void should_not_pick_blob_txs_with_MaxFeePerBlobGas_lower_than_CurrentPricePerBlobGas([Values(1, 2, 99, 100, 101, 1000)] int threshold) { _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); - const int currentPricePerDataGasInGwei = 250; - _headInfo.CurrentPricePerDataGas.Returns(currentPricePerDataGasInGwei.GWei()); + const int currentPricePerBlobGasInGwei = 250; + _headInfo.CurrentPricePerBlobGas.Returns(currentPricePerBlobGasInGwei.GWei()); int addedTxsCount = TestItem.PrivateKeys.Length; Transaction[] transactions = new Transaction[addedTxsCount]; @@ -404,7 +404,7 @@ public void should_not_pick_blob_txs_with_MaxFeePerDataGas_lower_than_CurrentPri { transactions[i] = Build.A.Transaction .WithShardBlobTxTypeAndFields() - .WithMaxFeePerDataGas(i.GWei()) + .WithMaxFeePerBlobGas(i.GWei()) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeys[i]) .TestObject; @@ -415,7 +415,7 @@ public void should_not_pick_blob_txs_with_MaxFeePerDataGas_lower_than_CurrentPri IList pickedTxs = _broadcaster.GetPersistentTxsToSend().HashesToSend; - int expectedCount = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount - currentPricePerDataGasInGwei); + int expectedCount = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount - currentPricePerBlobGasInGwei); pickedTxs.Count.Should().Be(expectedCount); List expectedTxs = new(); @@ -425,7 +425,7 @@ public void should_not_pick_blob_txs_with_MaxFeePerDataGas_lower_than_CurrentPri expectedTxs.Add(transactions[addedTxsCount - i]); } - expectedTxs.Count(t => t.MaxFeePerDataGas >= (UInt256)currentPricePerDataGasInGwei).Should().Be(expectedCount); + expectedTxs.Count(t => t.MaxFeePerBlobGas >= (UInt256)currentPricePerBlobGasInGwei).Should().Be(expectedCount); } [Test] diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index dcd9bace8c5..f39b551e2c1 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -644,7 +644,7 @@ public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, t Transaction firstTransaction = Build.A.Transaction .WithShardBlobTxTypeAndFields() - .WithMaxFeePerDataGas(UInt256.Zero) + .WithMaxFeePerBlobGas(UInt256.Zero) .WithNonce(UInt256.Zero) .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) @@ -653,7 +653,7 @@ public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, t Transaction transactionWithPotentialOverflow = Build.A.Transaction .WithShardBlobTxTypeAndFields() - .WithMaxFeePerDataGas(supportsBlobs + .WithMaxFeePerBlobGas(supportsBlobs ? UInt256.One : UInt256.Zero) .WithNonce(UInt256.One) @@ -1669,13 +1669,13 @@ public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - _headInfo.CurrentPricePerDataGas = UInt256.MaxValue; + _headInfo.CurrentPricePerBlobGas = UInt256.MaxValue; Transaction tx = Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithMaxFeePerGas((UInt256)maxPriorityFeePerGas) .WithMaxPriorityFeePerGas((UInt256)maxPriorityFeePerGas) - .WithMaxFeePerDataGas(UInt256.One) + .WithMaxFeePerBlobGas(UInt256.One) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(expectedResult @@ -1873,7 +1873,7 @@ public void should_remove_replaced_blob_tx() .WithNonce(UInt256.Zero) .WithMaxFeePerGas(oldTx.MaxFeePerGas * 2) .WithMaxPriorityFeePerGas(oldTx.MaxPriorityFeePerGas * 2) - .WithMaxFeePerDataGas(oldTx.MaxFeePerDataGas * 2) + .WithMaxFeePerBlobGas(oldTx.MaxFeePerBlobGas * 2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; @@ -1911,7 +1911,7 @@ public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_st .WithNonce(UInt256.Zero) .WithMaxFeePerGas(1.GWei()) .WithMaxPriorityFeePerGas(1.GWei()) - .WithMaxFeePerDataGas(UInt256.One) + .WithMaxFeePerBlobGas(UInt256.One) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); @@ -1929,7 +1929,7 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - _headInfo.CurrentPricePerDataGas = UInt256.MaxValue; + _headInfo.CurrentPricePerBlobGas = UInt256.MaxValue; Transaction tx = Build.A.Transaction .WithType(isBlob ? TxType.Blob : TxType.EIP1559) @@ -1937,7 +1937,7 @@ public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_ .WithNonce(UInt256.Zero) .WithMaxFeePerGas(1.GWei()) .WithMaxPriorityFeePerGas(1.GWei()) - .WithMaxFeePerDataGas(isBlob ? UInt256.One : null) + .WithMaxFeePerBlobGas(isBlob ? UInt256.One : null) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); @@ -1979,7 +1979,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, .WithNonce(UInt256.Zero) .WithMaxFeePerGas(firstTx.MaxFeePerGas * 2) .WithMaxPriorityFeePerGas(firstTx.MaxPriorityFeePerGas * 2) - .WithMaxFeePerDataGas(firstTx.MaxFeePerDataGas * 2) + .WithMaxFeePerBlobGas(firstTx.MaxFeePerBlobGas * 2) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 2aece93653f..cfbcea9346e 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -10,7 +10,7 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { - protected const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxDataGasPerBlock / Eip4844Constants.DataGasPerBlob); + protected const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxBlobGasPerBlock / Eip4844Constants.BlobGasPerBlob); public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, logManager) diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index d708f92e275..243543864ba 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -32,7 +32,7 @@ public int Compare(Transaction? x, Transaction? y) if (y.MaxFeePerGas * 2 > x.MaxFeePerGas) return 1; if (y.MaxPriorityFeePerGas * 2 > x.MaxPriorityFeePerGas) return 1; - if (y.MaxFeePerDataGas * 2 > x.MaxFeePerDataGas) return 1; + if (y.MaxFeePerBlobGas * 2 > x.MaxFeePerBlobGas) return 1; // if we are here, it means that all new fees are at least 2x higher than old ones, so replacement is allowed return -1; diff --git a/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs b/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs index 0ba937d8169..dc4de5f6b43 100644 --- a/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs +++ b/src/Nethermind/Nethermind.TxPool/IChainHeadInfoProvider.cs @@ -18,7 +18,7 @@ public interface IChainHeadInfoProvider public UInt256 CurrentBaseFee { get; } - public UInt256 CurrentPricePerDataGas { get; } + public UInt256 CurrentPricePerBlobGas { get; } event EventHandler HeadChanged; } diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 75578d679de..8e8c244ee59 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -21,7 +21,7 @@ public LightTransaction(Transaction fullTx) GasLimit = fullTx.GasLimit; GasPrice = fullTx.GasPrice; // means MaxPriorityFeePerGas DecodedMaxFeePerGas = fullTx.DecodedMaxFeePerGas; - MaxFeePerDataGas = fullTx.MaxFeePerDataGas; + MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; GasBottleneck = fullTx.GasBottleneck; } diff --git a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs index fe0d6cb845f..f5aeaea67f9 100644 --- a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs @@ -74,8 +74,8 @@ internal static bool IsOverflowWhenAddingTxCostToCumulative(this Transaction tx, if (tx.SupportsBlobs) { // if tx.SupportsBlobs and has BlobVersionedHashes = null, it will throw on earlier step of validation, in TxValidator - overflow |= UInt256.MultiplyOverflow(Eip4844Constants.DataGasPerBlob, (UInt256)tx.BlobVersionedHashes!.Length, out UInt256 dataGas); - overflow |= UInt256.MultiplyOverflow(dataGas, tx.MaxFeePerDataGas ?? UInt256.MaxValue, out UInt256 dataGasCost); + overflow |= UInt256.MultiplyOverflow(Eip4844Constants.BlobGasPerBlob, (UInt256)tx.BlobVersionedHashes!.Length, out UInt256 dataGas); + overflow |= UInt256.MultiplyOverflow(dataGas, tx.MaxFeePerBlobGas ?? UInt256.MaxValue, out UInt256 dataGasCost); overflow |= UInt256.AddOverflow(cumulativeCost, dataGasCost, out cumulativeCost); } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index ea9c03216f2..3e8d4ff0c87 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -192,7 +192,7 @@ public void BroadcastPersistentTxs() } else { - if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) // for not-blob tx MaxFeePerDataGas + if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) // for not-blob tx MaxFeePerDataGas { // is null so check will be skipped continue; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 02bb2386e7d..e757dc02d50 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -467,7 +467,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe previousTxBottleneck ??= tx.CalculateAffordableGasPrice(_specProvider.GetCurrentHeadSpec().IsEip1559Enabled, _headInfo.CurrentBaseFee, balance); - if (tx.MaxFeePerDataGas < _headInfo.CurrentPricePerDataGas) + if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) { gasBottleneck = UInt256.Zero; } From ccc93e0ac9ad11b249f0256fc42f86f98f491645 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 19:01:39 +0200 Subject: [PATCH 058/148] cosmetics --- .../ChainHeadInfoProvider.cs | 4 ++-- .../Nethermind.TxPool.Test/TxPoolTests.cs | 15 ++++++--------- .../PersistentBlobTxDistinctSortedPool.cs | 2 +- .../Nethermind.TxPool/ITxPoolConfig.cs | 4 ++-- .../Nethermind.TxPool/TxBroadcaster.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 19 +++++++++---------- .../Nethermind.TxPool/TxPoolConfig.cs | 4 ++-- 7 files changed, 23 insertions(+), 27 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs index b3400e2ab5f..c23aeed1f73 100644 --- a/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs +++ b/src/Nethermind/Nethermind.Blockchain/ChainHeadInfoProvider.cs @@ -52,8 +52,8 @@ private void OnHeadChanged(object? sender, BlockReplacementEventArgs e) BlockGasLimit = e.Block!.GasLimit; CurrentBaseFee = e.Block.Header.BaseFeePerGas; CurrentPricePerBlobGas = - BlobGasCalculator.TryCalculateBlobGasPricePerUnit(e.Block.Header, out UInt256 currentPricePerDataGas) - ? currentPricePerDataGas + BlobGasCalculator.TryCalculateBlobGasPricePerUnit(e.Block.Header, out UInt256 currentPricePerBlobGas) + ? currentPricePerBlobGas : UInt256.Zero; HeadChanged?.Invoke(sender, e); } diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index f39b551e2c1..435878c025f 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1669,13 +1669,10 @@ public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - _headInfo.CurrentPricePerBlobGas = UInt256.MaxValue; - Transaction tx = Build.A.Transaction .WithShardBlobTxTypeAndFields() .WithMaxFeePerGas((UInt256)maxPriorityFeePerGas) .WithMaxPriorityFeePerGas((UInt256)maxPriorityFeePerGas) - .WithMaxFeePerBlobGas(UInt256.One) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(expectedResult @@ -1812,7 +1809,7 @@ public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() [Test] public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1854,9 +1851,9 @@ public void should_not_throw_when_asking_for_non_existing_tx() } [Test] - public void should_remove_replaced_blob_tx() + public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = true }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = true }; BlobTxStorage blobTxStorage = new(new MemDb()); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1902,7 +1899,7 @@ public void should_remove_replaced_blob_tx() [Test] public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_storage_enabled([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -1923,9 +1920,9 @@ public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_st } [Test] - public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerDataGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) + public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerBlobGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobPoolEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 143e3cd636e..fd3e22d0195 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -17,7 +17,7 @@ public class PersistentBlobTxDistinctSortedPool : BlobTxDistinctSortedPool private readonly ILogger _logger; public PersistentBlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfig txPoolConfig, IComparer comparer, ILogManager logManager) - : base(txPoolConfig.PersistentBlobPoolSize, comparer, logManager) + : base(txPoolConfig.PersistentBlobStorageSize, comparer, logManager) { _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _blobTxCache = new(txPoolConfig.BlobCacheSize, txPoolConfig.BlobCacheSize, "blob txs cache"); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index a7c3a1d8091..f069bf20307 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -14,10 +14,10 @@ public interface ITxPoolConfig : IConfig int Size { get; set; } [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] - bool PersistentBlobPoolEnabled { get; set; } + bool PersistentBlobStorageEnabled { get; set; } [ConfigItem(DefaultValue = "131072", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] - int PersistentBlobPoolSize { get; set; } + int PersistentBlobStorageSize { get; set; } [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage")] int BlobCacheSize { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index 3e8d4ff0c87..b1f251e1b77 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -192,7 +192,7 @@ public void BroadcastPersistentTxs() } else { - if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) // for not-blob tx MaxFeePerDataGas + if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) // for not-blob tx MaxFeePerBlobGas { // is null so check will be skipped continue; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e757dc02d50..bd19891426a 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -99,7 +99,7 @@ public TxPool(IEthereumEcdsa ecdsa, AddNodeInfoEntryForTxPool(); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - _blobTransactions = txPoolConfig.PersistentBlobPoolEnabled + _blobTransactions = txPoolConfig.PersistentBlobStorageEnabled ? new PersistentBlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager) : new BlobTxDistinctSortedPool(_txPoolConfig.InMemoryBlobPoolSize, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); @@ -166,10 +166,6 @@ public Transaction[] GetPendingTransactionsBySender(Address address) => public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; - - // public Transaction[] GetPendingBlobTransactionsBySender(Address address) => - // _blobTransactions.GetBucketSnapshot(address); - private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { try @@ -500,11 +496,14 @@ private void UpdateBuckets() _transactions.UpdatePool(_accounts, _updateBucket); // ensure the capacity of the blob pool - if (_blobTransactions.Count > _txPoolConfig.PersistentBlobPoolSize) - if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.PersistentBlobPoolSize}"); - - if (_blobTransactions.Count == _txPoolConfig.PersistentBlobPoolSize) - if (_logger.IsDebug) _logger.Debug($"Blob TxPool has reached max size of {_txPoolConfig.PersistentBlobPoolSize}, blob txs can be evicted now"); + if (_blobTransactions.Count > (_txPoolConfig.PersistentBlobStorageEnabled + ? _txPoolConfig.PersistentBlobStorageSize + : _txPoolConfig.InMemoryBlobPoolSize)) + if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.PersistentBlobStorageSize}"); + + if (_txPoolConfig.PersistentBlobStorageEnabled + && _blobTransactions.Count == _txPoolConfig.PersistentBlobStorageSize) + if (_logger.IsDebug) _logger.Debug($"Blob persistent storage has reached max size of {_txPoolConfig.PersistentBlobStorageSize}, blob txs can be evicted now"); _blobTransactions.UpdatePool(_accounts, _updateBucket); } diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 9b5efe627e7..6f0c01aeca8 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,8 +7,8 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; - public bool PersistentBlobPoolEnabled { get; set; } = false; - public int PersistentBlobPoolSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice + public bool PersistentBlobStorageEnabled { get; set; } = false; + public int PersistentBlobStorageSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice public int BlobCacheSize { get; set; } = 256; public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled public int HashCacheSize { get; set; } = 512 * 1024; From 3d3fe142558cbafebe972027e4f1b5c003cb96f5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 27 Jul 2023 20:25:56 +0200 Subject: [PATCH 059/148] fix bug in broadcasting - use size of full blob tx in light tx --- src/Nethermind/Nethermind.Core/Transaction.cs | 5 ++-- .../TxBroadcasterTests.cs | 28 +++++++++++++++++++ .../Nethermind.TxPool/LightTransaction.cs | 1 + 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index 50e1283139f..dff9ccb1a57 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -130,13 +130,14 @@ private void ClearPreHashInternal() /// Used for sorting in edge cases. public ulong PoolIndex { get; set; } - private int? _size = null; + protected int? Size { get; set; } = null; + /// /// Encoded transaction length /// public int GetLength(ITransactionSizeCalculator sizeCalculator) { - return _size ??= sizeCalculator.GetLength(this); + return Size ??= sizeCalculator.GetLength(this); } public string ToShortString() diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index 7f855aa8f1b..dcdf8f65fa6 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -167,6 +167,34 @@ public void should_add_light_form_of_blob_txs_to_persistent_txs_but_not_return_i returnedTx.Should().BeEquivalentTo(isBlob ? null : tx); } + [Test] + public void should_announce_details_of_full_blob_tx() + { + _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); + + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .SignedAndResolved().TestObject; + + Transaction lightTx = new LightTransaction(tx); + + int size = tx.GetLength(); + size.Should().Be(131324); + lightTx.GetLength().Should().Be(size); + + _broadcaster.Broadcast(tx, true); + _broadcaster.GetSnapshot().Length.Should().Be(1); + _broadcaster.GetSnapshot().FirstOrDefault().Should().BeEquivalentTo(lightTx); + + ITxPoolPeer peer = Substitute.For(); + peer.Id.Returns(TestItem.PublicKeyA); + + _broadcaster.AddPeer(peer); + _broadcaster.BroadcastPersistentTxs(); + + peer.Received(1).SendNewTransactions(Arg.Is>(t => t.FirstOrDefault().GetLength() == size), false); + } + [Test] public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast([Values(1, 2, 25, 50, 99, 100, 101, 1000)] int threshold) { diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 8e8c244ee59..eb5362d92c2 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -24,5 +24,6 @@ public LightTransaction(Transaction fullTx) MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; GasBottleneck = fullTx.GasBottleneck; + Size = fullTx.GetLength(); } } From 78de5bebf9878b4e81bc36282c903825de87918e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 28 Jul 2023 12:01:27 +0200 Subject: [PATCH 060/148] cosmetics --- src/Nethermind/Nethermind.Core/Transaction.cs | 4 ++-- src/Nethermind/Nethermind.TxPool/LightTransaction.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Transaction.cs b/src/Nethermind/Nethermind.Core/Transaction.cs index dff9ccb1a57..4dabf00a3fe 100644 --- a/src/Nethermind/Nethermind.Core/Transaction.cs +++ b/src/Nethermind/Nethermind.Core/Transaction.cs @@ -130,14 +130,14 @@ private void ClearPreHashInternal() /// Used for sorting in edge cases. public ulong PoolIndex { get; set; } - protected int? Size { get; set; } = null; + protected int? _size = null; /// /// Encoded transaction length /// public int GetLength(ITransactionSizeCalculator sizeCalculator) { - return Size ??= sizeCalculator.GetLength(this); + return _size ??= sizeCalculator.GetLength(this); } public string ToShortString() diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index eb5362d92c2..c37375cae24 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -24,6 +24,6 @@ public LightTransaction(Transaction fullTx) MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; GasBottleneck = fullTx.GasBottleneck; - Size = fullTx.GetLength(); + _size = fullTx.GetLength(); } } From e515f32ac4cbeaae1cf0d9dc8664230f9047ba5c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 3 Aug 2023 12:29:50 +0200 Subject: [PATCH 061/148] lower default PersistentBlobStorageSize --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index f069bf20307..5875e6d2c68 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -16,7 +16,7 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] bool PersistentBlobStorageEnabled { get; set; } - [ConfigItem(DefaultValue = "131072", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] + [ConfigItem(DefaultValue = "16384", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] int PersistentBlobStorageSize { get; set; } [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage")] diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 6f0c01aeca8..6acca9f2664 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -8,7 +8,9 @@ public class TxPoolConfig : ITxPoolConfig public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; public bool PersistentBlobStorageEnabled { get; set; } = false; - public int PersistentBlobStorageSize { get; set; } = 128 * 1024; // we need some limit of blob txs, but extremely high to be limitless in practice + public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 12,5GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); + // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. + // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) public int BlobCacheSize { get; set; } = 256; public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled public int HashCacheSize { get; set; } = 512 * 1024; From 58fc2fd8e266ae5af33817e98a106a8797511e30 Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Fri, 11 Aug 2023 17:36:24 +0200 Subject: [PATCH 062/148] Divide TxPoolTests --- .../TxPoolTests.Blobs.cs | 352 ++++++++++++++++++ .../Nethermind.TxPool.Test/TxPoolTests.cs | 314 +--------------- 2 files changed, 353 insertions(+), 313 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs new file mode 100644 index 00000000000..03299fe7386 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -0,0 +1,352 @@ +// SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using FluentAssertions; +using Nethermind.Blockchain; +using Nethermind.Config; +using Nethermind.Consensus; +using Nethermind.Consensus.Comparers; +using Nethermind.Consensus.Transactions; +using Nethermind.Consensus.Validators; +using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; +using Nethermind.Core.Specs; +using Nethermind.Core.Test.Builders; +using Nethermind.Crypto; +using Nethermind.Db; +using Nethermind.Evm; +using Nethermind.Int256; +using Nethermind.Logging; +using Nethermind.Specs; +using Nethermind.Specs.Forks; +using Nethermind.Specs.Test; +using Nethermind.State; +using Nethermind.Trie.Pruning; +using Nethermind.TxPool.Filters; +using NSubstitute; +using NUnit.Framework; + +namespace Nethermind.TxPool.Test +{ + [TestFixture] + public partial class TxPoolTests + { + // blob collection is designed to be infinite, so there is no point in checking if is full and comparing + // incoming tx with the worst already pending one (candidate for evicting). There will be no eviction. + [Test] + public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); + + for (int i = 0; i < txPoolConfig.Size; i++) + { + Transaction tx = Build.A.Transaction + .WithNonce((UInt256)i) + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(1.GWei() + (UInt256)(100 - i)) + .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)(100 - i)) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); + } + + _txPool.GetPendingTransactionsCount().Should().Be(txPoolConfig.Size); + + Transaction feeTooLow1559Tx = Build.A.Transaction + .WithType(TxType.EIP1559) + .WithMaxFeePerGas(1.GWei() + UInt256.One) + .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; + + Transaction feeTooLowBlobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei() + UInt256.One) + .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; + + + _txPool.SubmitTx(feeTooLow1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.FeeTooLow); + _txPool.SubmitTx(feeTooLowBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + } + + [Test] + public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) + { + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + BlobTxStorage blobTxStorage = new(new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction blobTxAdded = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(blobTxAdded, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.TryGetPendingTransaction(blobTxAdded.Hash!, out Transaction blobTxReturned); + + blobTxReturned.Should().BeEquivalentTo(blobTxAdded); + + blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().Be(isPersistentStorage); // additional check for persistent db + if (isPersistentStorage) + { + blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex + } + } + + [Test] + public void should_not_throw_when_asking_for_non_existing_tx() + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + BlobTxStorage blobTxStorage = new(new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + + _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); + blobTxReturned.Should().BeNull(); + + blobTxStorage.TryGet(TestItem.KeccakA, out Transaction blobTxFromDb).Should().BeFalse(); + blobTxFromDb.Should().BeNull(); + } + + [TestCase(999_999_999, false)] + [TestCase(1_000_000_000, true)] + public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than_1GWei(int maxPriorityFeePerGas, bool expectedResult) + { + TxPoolConfig txPoolConfig = new() { Size = 10 }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas((UInt256)maxPriorityFeePerGas) + .WithMaxPriorityFeePerGas((UInt256)maxPriorityFeePerGas) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(expectedResult + ? AcceptTxResult.Accepted + : AcceptTxResult.FeeTooLow); + } + + [Test] + public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) + { + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(UInt256.Zero) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction nonceGapTx = Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce((UInt256)2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(nonceGapTx, TxHandlingOptions.None).Should().Be(isBlob ? AcceptTxResult.NonceGap : AcceptTxResult.Accepted); + } + + [Test] + public void should_not_allow_to_have_pending_transactions_of_both_blob_type_and_other([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) + { + Transaction GetTx(bool isBlob, UInt256 nonce) + { + return Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); + Transaction secondTx = GetTx(secondIsBlob, UInt256.One); + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted); + } + + [Test] + public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() + { + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = true }; + BlobTxStorage blobTxStorage = new(new MemDb()); + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction oldTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction newTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(oldTx.MaxFeePerGas * 2) + .WithMaxPriorityFeePerGas(oldTx.MaxPriorityFeePerGas * 2) + .WithMaxFeePerBlobGas(oldTx.MaxFeePerBlobGas * 2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + + _txPool.SubmitTx(oldTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(oldTx.Hash!, out Transaction blobTxReturned).Should().BeTrue(); + blobTxReturned.Should().BeEquivalentTo(oldTx); + blobTxStorage.TryGet(oldTx.Hash, out Transaction blobTxFromDb).Should().BeTrue(); + blobTxFromDb.Should().BeEquivalentTo(oldTx, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex + + _txPool.SubmitTx(newTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); + blobTxReturned.Should().BeEquivalentTo(newTx); + blobTxStorage.TryGet(oldTx.Hash, out blobTxFromDb).Should().BeFalse(); + blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); + blobTxFromDb.Should().BeEquivalentTo(newTx, options => options + .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... + .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... + .Excluding(t => t.PoolIndex)); // ...and PoolIndex + } + + [Test] + public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_storage_enabled([Values(true, false)] bool isPersistentStorage) + { + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction tx = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerBlobGas(UInt256.One) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.GetPendingTransactionsCount().Should().Be(0); + + _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); + returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx); + } + + [Test] + public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerBlobGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) + { + TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + _headInfo.CurrentPricePerBlobGas = UInt256.MaxValue; + + Transaction tx = Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithMaxFeePerBlobGas(isBlob ? UInt256.One : null) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingBlobTransactionsCount().Should().Be(isBlob ? 1 : 0); + _txPool.GetPendingTransactionsCount().Should().Be(isBlob ? 0 : 1); + if (isBlob) + { + _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); + returned.GasBottleneck.Should().Be(UInt256.Zero); + returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx, + options => options.Excluding(t => t.GasBottleneck)); + returned.Should().NotBeEquivalentTo(isPersistentStorage ? tx : new LightTransaction(tx)); + } + else + { + _txPool.TryGetPendingTransaction(tx.Hash!, out Transaction eip1559Tx); + eip1559Tx.Should().BeEquivalentTo(tx); + eip1559Tx.GasBottleneck.Should().Be(1.GWei()); + } + } + + [Test] + public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, 2, 3, 4, 5, 6)] int blobsInFirstTx, [Values(1, 2, 3, 4, 5, 6)] int blobsInSecondTx) + { + bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; + + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(blobsInFirstTx) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + Transaction secondTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(blobsInSecondTx) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(firstTx.MaxFeePerGas * 2) + .WithMaxPriorityFeePerGas(firstTx.MaxPriorityFeePerGas * 2) + .WithMaxFeePerBlobGas(firstTx.MaxFeePerBlobGas * 2) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(shouldReplace ? AcceptTxResult.Accepted : AcceptTxResult.ReplacementNotAllowed); + _txPool.GetPendingBlobTransactionsCount().Should().Be(1); + _txPool.TryGetPendingTransaction(firstTx.Hash!, out Transaction returnedFirstTx).Should().Be(!shouldReplace); + _txPool.TryGetPendingTransaction(secondTx.Hash!, out Transaction returnedSecondTx).Should().Be(shouldReplace); + returnedFirstTx.Should().BeEquivalentTo(shouldReplace ? null : firstTx); + returnedSecondTx.Should().BeEquivalentTo(shouldReplace ? secondTx : null); + } + + [TestCase(0, 97)] + [TestCase(1, 131324)] + [TestCase(2, 262534)] + [TestCase(3, 393741)] + [TestCase(4, 524947)] + [TestCase(5, 656156)] + [TestCase(6, 787365)] + public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int expectedLength) + { + Transaction blobTx = Build.A.Transaction + .WithShardBlobTxTypeAndFields(numberOfBlobs) + .SignedAndResolved() + .TestObject; + blobTx.GetLength().Should().Be(expectedLength); + } + } +} diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index b8a31b0e22f..d8bc646256d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -35,7 +35,7 @@ namespace Nethermind.TxPool.Test { [TestFixture] - public class TxPoolTests + public partial class TxPoolTests { private ILogManager _logManager; private IEthereumEcdsa _ethereumEcdsa; @@ -1667,75 +1667,6 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); } - [TestCase(999_999_999, false)] - [TestCase(1_000_000_000, true)] - public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than_1GWei(int maxPriorityFeePerGas, bool expectedResult) - { - TxPoolConfig txPoolConfig = new() { Size = 10 }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction tx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas((UInt256)maxPriorityFeePerGas) - .WithMaxPriorityFeePerGas((UInt256)maxPriorityFeePerGas) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(expectedResult - ? AcceptTxResult.Accepted - : AcceptTxResult.FeeTooLow); - } - - [Test] - public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) - { - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction firstTx = Build.A.Transaction - .WithType(isBlob ? TxType.Blob : TxType.EIP1559) - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - Transaction nonceGapTx = Build.A.Transaction - .WithType(isBlob ? TxType.Blob : TxType.EIP1559) - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce((UInt256)2) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(nonceGapTx, TxHandlingOptions.None).Should().Be(isBlob ? AcceptTxResult.NonceGap : AcceptTxResult.Accepted); - } - - [Test] - public void should_not_allow_to_have_pending_transactions_of_both_blob_type_and_other([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) - { - Transaction GetTx(bool isBlob, UInt256 nonce) - { - return Build.A.Transaction - .WithType(isBlob ? TxType.Blob : TxType.EIP1559) - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce(nonce) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - } - - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); - Transaction secondTx = GetTx(secondIsBlob, UInt256.One); - - _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted); - } - [Test] public async Task should_allow_to_have_pending_transaction_of_other_type_if_conflicting_one_was_included([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) { @@ -1770,249 +1701,6 @@ Transaction GetTx(bool isBlob, UInt256 nonce) _txPool.GetPendingBlobTransactionsCount().Should().Be(secondIsBlob ? 1 : 0); } - // blob collection is designed to be infinite, so there is no point in checking if is full and comparing - // incoming tx with the worst already pending one (candidate for evicting). There will be no eviction. - [Test] - public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() - { - TxPoolConfig txPoolConfig = new() { Size = 10 }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); - - for (int i = 0; i < txPoolConfig.Size; i++) - { - Transaction tx = Build.A.Transaction - .WithNonce((UInt256)i) - .WithType(TxType.EIP1559) - .WithMaxFeePerGas(1.GWei() + (UInt256)(100 - i)) - .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)(100 - i)) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); - } - - _txPool.GetPendingTransactionsCount().Should().Be(txPoolConfig.Size); - - Transaction feeTooLow1559Tx = Build.A.Transaction - .WithType(TxType.EIP1559) - .WithMaxFeePerGas(1.GWei() + UInt256.One) - .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; - - Transaction feeTooLowBlobTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(1.GWei() + UInt256.One) - .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; - - - _txPool.SubmitTx(feeTooLow1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.FeeTooLow); - _txPool.SubmitTx(feeTooLowBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - } - - [Test] - public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) - { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; - BlobTxStorage blobTxStorage = new(new MemDb()); - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction blobTxAdded = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(blobTxAdded, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.TryGetPendingTransaction(blobTxAdded.Hash!, out Transaction blobTxReturned); - - blobTxReturned.Should().BeEquivalentTo(blobTxAdded); - - blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().Be(isPersistentStorage); // additional check for persistent db - if (isPersistentStorage) - { - blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex - } - } - - [Test] - public void should_not_throw_when_asking_for_non_existing_tx() - { - TxPoolConfig txPoolConfig = new() { Size = 10 }; - BlobTxStorage blobTxStorage = new(new MemDb()); - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); - - _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); - blobTxReturned.Should().BeNull(); - - blobTxStorage.TryGet(TestItem.KeccakA, out Transaction blobTxFromDb).Should().BeFalse(); - blobTxFromDb.Should().BeNull(); - } - - [Test] - public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() - { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = true }; - BlobTxStorage blobTxStorage = new(new MemDb()); - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction oldTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - Transaction newTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(oldTx.MaxFeePerGas * 2) - .WithMaxPriorityFeePerGas(oldTx.MaxPriorityFeePerGas * 2) - .WithMaxFeePerBlobGas(oldTx.MaxFeePerBlobGas * 2) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - - _txPool.SubmitTx(oldTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - _txPool.TryGetPendingTransaction(oldTx.Hash!, out Transaction blobTxReturned).Should().BeTrue(); - blobTxReturned.Should().BeEquivalentTo(oldTx); - blobTxStorage.TryGet(oldTx.Hash, out Transaction blobTxFromDb).Should().BeTrue(); - blobTxFromDb.Should().BeEquivalentTo(oldTx, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex - - _txPool.SubmitTx(newTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); - blobTxReturned.Should().BeEquivalentTo(newTx); - blobTxStorage.TryGet(oldTx.Hash, out blobTxFromDb).Should().BeFalse(); - blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); - blobTxFromDb.Should().BeEquivalentTo(newTx, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex - } - - [Test] - public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_storage_enabled([Values(true, false)] bool isPersistentStorage) - { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction tx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithMaxFeePerBlobGas(UInt256.One) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - _txPool.GetPendingTransactionsCount().Should().Be(0); - - _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); - returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx); - } - - [Test] - public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerBlobGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) - { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - _headInfo.CurrentPricePerBlobGas = UInt256.MaxValue; - - Transaction tx = Build.A.Transaction - .WithType(isBlob ? TxType.Blob : TxType.EIP1559) - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithMaxFeePerBlobGas(isBlob ? UInt256.One : null) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingBlobTransactionsCount().Should().Be(isBlob ? 1 : 0); - _txPool.GetPendingTransactionsCount().Should().Be(isBlob ? 0 : 1); - if (isBlob) - { - _txPool.TryGetBlobTxSortingEquivalent(tx.Hash!, out Transaction returned); - returned.GasBottleneck.Should().Be(UInt256.Zero); - returned.Should().BeEquivalentTo(isPersistentStorage ? new LightTransaction(tx) : tx, - options => options.Excluding(t => t.GasBottleneck)); - returned.Should().NotBeEquivalentTo(isPersistentStorage ? tx : new LightTransaction(tx)); - } - else - { - _txPool.TryGetPendingTransaction(tx.Hash!, out Transaction eip1559Tx); - eip1559Tx.Should().BeEquivalentTo(tx); - eip1559Tx.GasBottleneck.Should().Be(1.GWei()); - } - } - - [Test] - public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, 2, 3, 4, 5, 6)] int blobsInFirstTx, [Values(1, 2, 3, 4, 5, 6)] int blobsInSecondTx) - { - bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; - - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction firstTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields(blobsInFirstTx) - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - Transaction secondTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields(blobsInSecondTx) - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(firstTx.MaxFeePerGas * 2) - .WithMaxPriorityFeePerGas(firstTx.MaxPriorityFeePerGas * 2) - .WithMaxFeePerBlobGas(firstTx.MaxFeePerBlobGas * 2) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - - _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - - _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(shouldReplace ? AcceptTxResult.Accepted : AcceptTxResult.ReplacementNotAllowed); - _txPool.GetPendingBlobTransactionsCount().Should().Be(1); - _txPool.TryGetPendingTransaction(firstTx.Hash!, out Transaction returnedFirstTx).Should().Be(!shouldReplace); - _txPool.TryGetPendingTransaction(secondTx.Hash!, out Transaction returnedSecondTx).Should().Be(shouldReplace); - returnedFirstTx.Should().BeEquivalentTo(shouldReplace ? null : firstTx); - returnedSecondTx.Should().BeEquivalentTo(shouldReplace ? secondTx : null); - } - - [TestCase(0, 97)] - [TestCase(1, 131324)] - [TestCase(2, 262534)] - [TestCase(3, 393741)] - [TestCase(4, 524947)] - [TestCase(5, 656156)] - [TestCase(6, 787365)] - public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int expectedLength) - { - Transaction blobTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields(numberOfBlobs) - .SignedAndResolved() - .TestObject; - blobTx.GetLength().Should().Be(expectedLength); - } - private IDictionary GetPeers(int limit = 100) { var peers = new Dictionary(); From edd4fca80f4bf6339230a3e267baa422dca93e5e Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Fri, 11 Aug 2023 17:37:53 +0200 Subject: [PATCH 063/148] cosmetic - usings --- .../TxPoolTests.Blobs.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 03299fe7386..6f67575066c 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -1,35 +1,12 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; -using System.Collections.Generic; -using System.Linq; -using System.Threading; -using System.Threading.Tasks; using FluentAssertions; -using Nethermind.Blockchain; -using Nethermind.Config; -using Nethermind.Consensus; -using Nethermind.Consensus.Comparers; -using Nethermind.Consensus.Transactions; -using Nethermind.Consensus.Validators; using Nethermind.Core; -using Nethermind.Core.Crypto; using Nethermind.Core.Extensions; -using Nethermind.Core.Specs; using Nethermind.Core.Test.Builders; -using Nethermind.Crypto; using Nethermind.Db; -using Nethermind.Evm; using Nethermind.Int256; -using Nethermind.Logging; -using Nethermind.Specs; -using Nethermind.Specs.Forks; -using Nethermind.Specs.Test; -using Nethermind.State; -using Nethermind.Trie.Pruning; -using Nethermind.TxPool.Filters; -using NSubstitute; using NUnit.Framework; namespace Nethermind.TxPool.Test From f8801b98c246971db3b3b0f624733f9b84245e24 Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Fri, 11 Aug 2023 17:55:48 +0200 Subject: [PATCH 064/148] more tests moved to TxPoolTests.Blobs --- .../TxPoolTests.Blobs.cs | 67 +++++++++++++++++++ .../Nethermind.TxPool.Test/TxPoolTests.cs | 65 ------------------ 2 files changed, 67 insertions(+), 65 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 6f67575066c..03f0c922504 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -1,11 +1,13 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; using Nethermind.Db; +using Nethermind.Evm; using Nethermind.Int256; using NUnit.Framework; @@ -310,6 +312,71 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, returnedSecondTx.Should().BeEquivalentTo(shouldReplace ? secondTx : null); } + [Test] + public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, true)] bool supportsBlobs) + { + _txPool = CreatePool(null, GetCancunSpecProvider()); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + UInt256.MaxValue.Divide(GasCostOf.Transaction * 2, out UInt256 halfOfMaxGasPriceWithoutOverflow); + + Transaction firstTransaction = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerBlobGas(UInt256.Zero) + .WithNonce(UInt256.Zero) + .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(firstTransaction, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); + + Transaction transactionWithPotentialOverflow = Build.A.Transaction + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerBlobGas(supportsBlobs + ? UInt256.One + : UInt256.Zero) + .WithNonce(UInt256.One) + .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(transactionWithPotentialOverflow, TxHandlingOptions.PersistentBroadcast).Should().Be(supportsBlobs ? AcceptTxResult.Int256Overflow : AcceptTxResult.Accepted); + } + + [Test] + public async Task should_allow_to_have_pending_transaction_of_other_type_if_conflicting_one_was_included([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) + { + Transaction GetTx(bool isBlob, UInt256 nonce) + { + return Build.A.Transaction + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .WithNonce(nonce) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + } + + _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); + Transaction secondTx = GetTx(secondIsBlob, UInt256.One); + + _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + + _txPool.GetPendingTransactionsCount().Should().Be(firstIsBlob ? 0 : 1); + _txPool.GetPendingBlobTransactionsCount().Should().Be(firstIsBlob ? 1 : 0); + _stateProvider.IncrementNonce(TestItem.AddressA); + await RaiseBlockAddedToMainAndWaitForTransactions(1); + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(0); + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.GetPendingTransactionsCount().Should().Be(secondIsBlob ? 0 : 1); + _txPool.GetPendingBlobTransactionsCount().Should().Be(secondIsBlob ? 1 : 0); + } + [TestCase(0, 97)] [TestCase(1, 131324)] [TestCase(2, 262534)] diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index d8bc646256d..2011ea1ada0 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -639,37 +639,6 @@ public void should_discard_tx_because_of_overflow_of_cumulative_cost_of_this_tx_ _txPool.SubmitTx(transactions[2], TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Int256Overflow); } - [Test] - public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, true)] bool supportsBlobs) - { - _txPool = CreatePool(null, GetCancunSpecProvider()); - - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - UInt256.MaxValue.Divide(GasCostOf.Transaction * 2, out UInt256 halfOfMaxGasPriceWithoutOverflow); - - Transaction firstTransaction = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerBlobGas(UInt256.Zero) - .WithNonce(UInt256.Zero) - .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - _txPool.SubmitTx(firstTransaction, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); - - Transaction transactionWithPotentialOverflow = Build.A.Transaction - .WithShardBlobTxTypeAndFields() - .WithMaxFeePerBlobGas(supportsBlobs - ? UInt256.One - : UInt256.Zero) - .WithNonce(UInt256.One) - .WithMaxFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .WithMaxPriorityFeePerGas(halfOfMaxGasPriceWithoutOverflow) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - - _txPool.SubmitTx(transactionWithPotentialOverflow, TxHandlingOptions.PersistentBroadcast).Should().Be(supportsBlobs ? AcceptTxResult.Int256Overflow : AcceptTxResult.Accepted); - } - [Test] public async Task should_not_dump_GasBottleneck_of_all_txs_in_bucket_if_first_tx_in_bucket_has_insufficient_balance_but_has_old_nonce() { @@ -1667,40 +1636,6 @@ public void Should_not_add_underpaid_tx_even_if_lower_nonces_are_expensive(int g result.Should().Be(expectedResult ? AcceptTxResult.Accepted : AcceptTxResult.FeeTooLowToCompete); } - [Test] - public async Task should_allow_to_have_pending_transaction_of_other_type_if_conflicting_one_was_included([Values(true, false)] bool firstIsBlob, [Values(true, false)] bool secondIsBlob) - { - Transaction GetTx(bool isBlob, UInt256 nonce) - { - return Build.A.Transaction - .WithType(isBlob ? TxType.Blob : TxType.EIP1559) - .WithShardBlobTxTypeAndFieldsIfBlobTx() - .WithMaxFeePerGas(1.GWei()) - .WithMaxPriorityFeePerGas(1.GWei()) - .WithNonce(nonce) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; - } - - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); - EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); - - Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); - Transaction secondTx = GetTx(secondIsBlob, UInt256.One); - - _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - - _txPool.GetPendingTransactionsCount().Should().Be(firstIsBlob ? 0 : 1); - _txPool.GetPendingBlobTransactionsCount().Should().Be(firstIsBlob ? 1 : 0); - _stateProvider.IncrementNonce(TestItem.AddressA); - await RaiseBlockAddedToMainAndWaitForTransactions(1); - - _txPool.GetPendingTransactionsCount().Should().Be(0); - _txPool.GetPendingBlobTransactionsCount().Should().Be(0); - _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.GetPendingTransactionsCount().Should().Be(secondIsBlob ? 0 : 1); - _txPool.GetPendingBlobTransactionsCount().Should().Be(secondIsBlob ? 1 : 0); - } - private IDictionary GetPeers(int limit = 100) { var peers = new Dictionary(); From 001fa5c07326f3b603597a16e4bba1dc1ae091f4 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 15 Aug 2023 12:00:06 +0200 Subject: [PATCH 065/148] refactor ParityRpcModuleTests to reuse setup code --- .../Modules/ParityRpcModuleTests.cs | 101 ++++++------------ 1 file changed, 31 insertions(+), 70 deletions(-) diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index d8da55e184d..ff878cc4b03 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -1,7 +1,6 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System; using System.Collections.Generic; using System.IO; using System.Net; @@ -47,13 +46,17 @@ public class ParityRpcModuleTests { private IParityRpcModule _parityRpcModule = null!; private Signer _signerStore = null!; + private EthereumEcdsa _ethereumEcdsa = null!; + private ITxPool _txPool = null!; + private IBlockTree _blockTree = null!; + private IReceiptStorage _receiptStorage = null!; [SetUp] public void Initialize() { LimboLogs logger = LimboLogs.Instance; MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - EthereumEcdsa ethereumEcdsa = new(specProvider.ChainId, logger); + _ethereumEcdsa = new(specProvider.ChainId, logger); Peer peerA = SetUpPeerA(); //standard case Peer peerB = SetUpPeerB(); //Session is null @@ -68,46 +71,43 @@ public void Initialize() IDb blockDb = new MemDb(); IDb headerDb = new MemDb(); IDb blockInfoDb = new MemDb(); - IBlockTree blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, LimboLogs.Instance); + _blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, logger); ITransactionComparerProvider transactionComparerProvider = - new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, + new TransactionComparerProvider(specProvider, _blockTree); + _txPool = new TxPool.TxPool(_ethereumEcdsa, new BlobTxStorage(new MemDb()), - new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, stateProvider), + new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), LimboLogs.Instance, transactionComparerProvider.GetDefaultComparer()); - IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); + _receiptStorage = new InMemoryReceiptStorage(); _signerStore = new Signer(specProvider.ChainId, TestItem.PrivateKeyB, logger); - _parityRpcModule = new ParityRpcModule(ethereumEcdsa, txPool, blockTree, receiptStorage, - new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), _signerStore, - new MemKeyStore(new[] { TestItem.PrivateKeyA }, Environment.SpecialFolder.ApplicationData.ToString()), - MainnetSpecProvider.Instance, peerManager); + _parityRpcModule = CreateParityRpcModule(peerManager); int blockNumber = 2; - Transaction pendingTransaction = Build.A.Transaction.Signed(ethereumEcdsa, TestItem.PrivateKeyD, false) + Transaction pendingTransaction = Build.A.Transaction.Signed(_ethereumEcdsa, TestItem.PrivateKeyD, false) .WithSenderAddress(Address.FromNumber((UInt256)blockNumber)).TestObject; pendingTransaction.Signature!.V = 37; stateProvider.CreateAccount(pendingTransaction.SenderAddress!, UInt256.UInt128MaxValue); - txPool.SubmitTx(pendingTransaction, TxHandlingOptions.None); + _txPool.SubmitTx(pendingTransaction, TxHandlingOptions.None); blockNumber = 1; - Transaction transaction1 = Build.A.Transaction.Signed(ethereumEcdsa, TestItem.PrivateKeyD, false) + Transaction transaction1 = Build.A.Transaction.Signed(_ethereumEcdsa, TestItem.PrivateKeyD, false) .WithSenderAddress(Address.FromNumber((UInt256)blockNumber)) .WithNonce(100).TestObject; transaction1.Signature!.V = 37; stateProvider.CreateAccount(transaction1.SenderAddress!, UInt256.UInt128MaxValue); - var transaction2 = Build.A.Transaction.Signed(ethereumEcdsa, TestItem.PrivateKeyD, false) + var transaction2 = Build.A.Transaction.Signed(_ethereumEcdsa, TestItem.PrivateKeyD, false) .WithSenderAddress(Address.FromNumber((UInt256)blockNumber)) .WithNonce(120).TestObject; transaction2.Signature!.V = 37; - var transaction3 = Build.A.Transaction.Signed(ethereumEcdsa, TestItem.PrivateKeyD, false) + var transaction3 = Build.A.Transaction.Signed(_ethereumEcdsa, TestItem.PrivateKeyD, false) .WithSenderAddress(Address.FromNumber((UInt256)blockNumber)) .WithNonce(110).TestObject; transaction2.Signature.V = 37; @@ -116,8 +116,8 @@ public void Initialize() .WithStateRoot(new Keccak("0x1ef7300d8961797263939a3d29bbba4ccf1702fabf02d8ad7a20b454edb6fd2f")) .TestObject; - blockTree.SuggestBlock(genesis); - blockTree.UpdateMainChain(new[] { genesis }, true); + _blockTree.SuggestBlock(genesis); + _blockTree.UpdateMainChain(new[] { genesis }, true); Block previousBlock = genesis; Block block = Build.A.Block.WithNumber(blockNumber).WithParent(previousBlock) @@ -125,8 +125,8 @@ public void Initialize() .WithTransactions(transaction1, transaction2, transaction3) .TestObject; - blockTree.SuggestBlock(block); - blockTree.UpdateMainChain(new[] { block }, true); + _blockTree.SuggestBlock(block); + _blockTree.UpdateMainChain(new[] { block }, true); LogEntry[] logEntries = new[] { Build.A.LogEntry.TestObject }; @@ -178,7 +178,7 @@ public void Initialize() Logs = logEntries }; - receiptStorage.Insert(block, receipt1, receipt2, receipt3); + _receiptStorage.Insert(block, receipt1, receipt2, receipt3); } private static Peer SetUpPeerA() @@ -255,6 +255,14 @@ private static Peer SetUpPeerC() return peer; } + private IParityRpcModule CreateParityRpcModule(IPeerManager? peerManager = null) + { + return new ParityRpcModule(_ethereumEcdsa, _txPool, _blockTree, _receiptStorage, + new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), + _signerStore, new MemKeyStore(new[] { TestItem.PrivateKeyA }, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), + MainnetSpecProvider.Instance, peerManager ?? Substitute.For()); + } + [Test] public async Task parity_pendingTransactions() { @@ -340,34 +348,11 @@ public async Task parity_netPeers_standard_case() [Test] public async Task parity_netPeers_empty_ActivePeers() { - LimboLogs logger = LimboLogs.Instance; - MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - EthereumEcdsa ethereumEcdsa = new(specProvider.ChainId, logger); - IDb blockDb = new MemDb(); - IDb headerDb = new MemDb(); - IDb blockInfoDb = new MemDb(); - IBlockTree blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, LimboLogs.Instance); - - ITransactionComparerProvider transactionComparerProvider = - new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, - new BlobTxStorage(new MemDb()), - new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), blockTree, new WorldState(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), - new TxPoolConfig(), - new TxValidator(specProvider.ChainId), - LimboLogs.Instance, - transactionComparerProvider.GetDefaultComparer()); - - IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); - IPeerManager peerManager = Substitute.For(); peerManager.ActivePeers.Returns(new List { }); peerManager.ConnectedPeers.Returns(new List { new(new Node(TestItem.PublicKeyA, "111.1.1.1", 11111, true)) }); - IParityRpcModule parityRpcModule = new ParityRpcModule(ethereumEcdsa, txPool, blockTree, receiptStorage, - new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), - _signerStore, new MemKeyStore(new[] { TestItem.PrivateKeyA }, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), - MainnetSpecProvider.Instance, peerManager); + IParityRpcModule parityRpcModule = CreateParityRpcModule(peerManager); string serialized = await RpcTest.TestSerializedRequest(parityRpcModule, "parity_netPeers"); string expectedResult = "{\"jsonrpc\":\"2.0\",\"result\":{\"active\":0,\"connected\":1,\"max\":0,\"peers\":[]},\"id\":67}"; @@ -377,32 +362,8 @@ public async Task parity_netPeers_empty_ActivePeers() [Test] public async Task parity_netPeers_null_ActivePeers() { - LimboLogs logger = LimboLogs.Instance; - MainnetSpecProvider specProvider = MainnetSpecProvider.Instance; - EthereumEcdsa ethereumEcdsa = new(specProvider.ChainId, logger); + IParityRpcModule parityRpcModule = CreateParityRpcModule(); - IDb blockDb = new MemDb(); - IDb headerDb = new MemDb(); - IDb blockInfoDb = new MemDb(); - IBlockTree blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, LimboLogs.Instance); - ITransactionComparerProvider transactionComparerProvider = - new TransactionComparerProvider(specProvider, blockTree); - TxPool.TxPool txPool = new(ethereumEcdsa, - new BlobTxStorage(new MemDb()), - new ChainHeadInfoProvider(specProvider, blockTree, new StateReader(new TrieStore(new MemDb(), LimboLogs.Instance), new MemDb(), LimboLogs.Instance)), - new TxPoolConfig(), - new TxValidator(specProvider.ChainId), - LimboLogs.Instance, - transactionComparerProvider.GetDefaultComparer()); - - IReceiptStorage receiptStorage = new InMemoryReceiptStorage(); - - IPeerManager peerManager = Substitute.For(); - - IParityRpcModule parityRpcModule = new ParityRpcModule(ethereumEcdsa, txPool, blockTree, receiptStorage, - new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), - _signerStore, new MemKeyStore(new[] { TestItem.PrivateKeyA }, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), - MainnetSpecProvider.Instance, peerManager); string serialized = await RpcTest.TestSerializedRequest(parityRpcModule, "parity_netPeers"); string expectedResult = "{\"jsonrpc\":\"2.0\",\"result\":{\"active\":0,\"connected\":0,\"max\":0,\"peers\":[]},\"id\":67}"; Assert.That(serialized, Is.EqualTo(expectedResult)); From 0fbc7782079386cf7b52b561105223cb1ce1308e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 15 Aug 2023 12:17:25 +0200 Subject: [PATCH 066/148] cosmetic --- .../Modules/ParityRpcModuleTests.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index ff878cc4b03..c55a9f44d22 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -73,15 +73,13 @@ public void Initialize() IDb blockInfoDb = new MemDb(); _blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, logger); - ITransactionComparerProvider transactionComparerProvider = - new TransactionComparerProvider(specProvider, _blockTree); _txPool = new TxPool.TxPool(_ethereumEcdsa, new BlobTxStorage(new MemDb()), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), LimboLogs.Instance, - transactionComparerProvider.GetDefaultComparer()); + new TransactionComparerProvider(specProvider, _blockTree).GetDefaultComparer()); _receiptStorage = new InMemoryReceiptStorage(); @@ -257,10 +255,15 @@ private static Peer SetUpPeerC() private IParityRpcModule CreateParityRpcModule(IPeerManager? peerManager = null) { - return new ParityRpcModule(_ethereumEcdsa, _txPool, _blockTree, _receiptStorage, + return new ParityRpcModule(_ethereumEcdsa, + _txPool, + _blockTree, + _receiptStorage, new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), - _signerStore, new MemKeyStore(new[] { TestItem.PrivateKeyA }, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), - MainnetSpecProvider.Instance, peerManager ?? Substitute.For()); + _signerStore, + new MemKeyStore(new[] { TestItem.PrivateKeyA },Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), + MainnetSpecProvider.Instance, + peerManager ?? Substitute.For()); } [Test] From 185b22d36615dbc00024ab39d113989fbb19c62b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 15 Aug 2023 13:10:29 +0200 Subject: [PATCH 067/148] refactor picking txs for rebroadcasting/reannouncing --- .../TransactionExtensions.cs | 4 +++ .../Nethermind.TxPool/TxBroadcaster.cs | 30 ++++++++++--------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs index 6c9f6275f7d..f6a46565050 100644 --- a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs @@ -21,6 +21,10 @@ public static int GetLength(this Transaction tx) return tx.GetLength(_transactionSizeCalculator); } + public static bool CanPayBaseFee(this Transaction tx, UInt256 currentBaseFee) => tx.MaxFeePerGas >= currentBaseFee; + + public static bool CanPayForBlobGas(this Transaction tx, UInt256 currentPricePerBlobGas) => !tx.SupportsBlobs || tx.MaxFeePerBlobGas >= currentPricePerBlobGas; + public static bool CanBeBroadcast(this Transaction tx) => !tx.SupportsBlobs && tx.GetLength() <= MaxSizeOfTxForBroadcast; internal static UInt256 CalculateGasPrice(this Transaction tx, bool eip1559Enabled, in UInt256 baseFee) diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index b1f251e1b77..213885eb38d 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -183,24 +183,26 @@ public void BroadcastPersistentTxs() { if (numberOfPersistentTxsToBroadcast > 0) { - if (tx.MaxFeePerGas >= _headInfo.CurrentBaseFee) + if (!tx.CanPayBaseFee(_headInfo.CurrentBaseFee)) { - if (tx.CanBeBroadcast()) - { - persistentTxsToSend ??= new List(numberOfPersistentTxsToBroadcast); - persistentTxsToSend.Add(tx); - } - else + continue; + } + + if (tx.CanBeBroadcast()) + { + persistentTxsToSend ??= new List(numberOfPersistentTxsToBroadcast); + persistentTxsToSend.Add(tx); + } + else + { + if (!tx.CanPayForBlobGas(_headInfo.CurrentPricePerBlobGas)) { - if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) // for not-blob tx MaxFeePerBlobGas - { // is null so check will be skipped - continue; - } - persistentHashesToSend ??= new List(numberOfPersistentTxsToBroadcast); - persistentHashesToSend.Add(tx); + continue; } - numberOfPersistentTxsToBroadcast--; + persistentHashesToSend ??= new List(numberOfPersistentTxsToBroadcast); + persistentHashesToSend.Add(tx); } + numberOfPersistentTxsToBroadcast--; } else { From 06c3b53c6d10fa23ae9439c321aa38ffaf45b6c7 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 15 Aug 2023 14:20:43 +0200 Subject: [PATCH 068/148] Add MaxBlobsPerBlock to Eip4844Constants and use in several places instead of calculating --- .../Nethermind.Consensus/Producers/TxPoolTxSource.cs | 2 +- src/Nethermind/Nethermind.Core/Eip4844Constants.cs | 5 +++-- .../Collections/BlobTxDistinctSortedPool.cs | 8 +++----- .../Collections/PersistentBlobTxDistinctSortedPool.cs | 6 +++--- 4 files changed, 10 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 90de3869999..29ac38f46e1 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -64,7 +64,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi foreach (Transaction blobTx in blobTransactions) { - if (BlobGasCalculator.CalculateBlobGas(blobsCounter) == Eip4844Constants.MaxBlobGasPerBlock) + if (blobsCounter == Eip4844Constants.MaxBlobsPerBlock) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); break; diff --git a/src/Nethermind/Nethermind.Core/Eip4844Constants.cs b/src/Nethermind/Nethermind.Core/Eip4844Constants.cs index f64eb88ffc3..fbd82dd753d 100644 --- a/src/Nethermind/Nethermind.Core/Eip4844Constants.cs +++ b/src/Nethermind/Nethermind.Core/Eip4844Constants.cs @@ -9,9 +9,10 @@ public class Eip4844Constants { public const int MinBlobsPerTransaction = 1; + public const int MaxBlobsPerBlock = 6; public const ulong BlobGasPerBlob = 1 << 17; - public const ulong TargetBlobGasPerBlock = BlobGasPerBlob * 3; - public const ulong MaxBlobGasPerBlock = BlobGasPerBlob * 6; + public const ulong MaxBlobGasPerBlock = BlobGasPerBlob * MaxBlobsPerBlock; + public const ulong TargetBlobGasPerBlock = MaxBlobGasPerBlock / 2; public const ulong MaxBlobGasPerTransaction = MaxBlobGasPerBlock; public static readonly UInt256 BlobGasUpdateFraction = 3338477; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index cfbcea9346e..41b0bef4f59 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -10,8 +10,6 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { - protected const int MaxNumberOfBlobsInBlock = (int)(Eip4844Constants.MaxBlobGasPerBlock / Eip4844Constants.BlobGasPerBlob); - public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, logManager) { @@ -31,7 +29,7 @@ public virtual IEnumerable GetBlobTransactions() int pickedBlobs = 0; List? blobTxsToReadd = null; - while (pickedBlobs < MaxNumberOfBlobsInBlock) + while (pickedBlobs < Eip4844Constants.MaxBlobsPerBlock) { Transaction? bestTx = GetFirsts().Min; @@ -40,7 +38,7 @@ public virtual IEnumerable GetBlobTransactions() break; } - if (pickedBlobs + bestTx.BlobVersionedHashes.Length <= MaxNumberOfBlobsInBlock) + if (pickedBlobs + bestTx.BlobVersionedHashes.Length <= Eip4844Constants.MaxBlobsPerBlock) { yield return bestTx; pickedBlobs += bestTx.BlobVersionedHashes.Length; @@ -48,7 +46,7 @@ public virtual IEnumerable GetBlobTransactions() if (TryRemove(bestTx.Hash)) { - blobTxsToReadd ??= new(MaxNumberOfBlobsInBlock); + blobTxsToReadd ??= new(Eip4844Constants.MaxBlobsPerBlock); blobTxsToReadd.Add(bestTx!); } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index fd3e22d0195..af042f0ed73 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -76,7 +76,7 @@ public override IEnumerable GetBlobTransactions() int pickedBlobs = 0; List? blobTxsToReadd = null; - while (pickedBlobs < MaxNumberOfBlobsInBlock) + while (pickedBlobs < Eip4844Constants.MaxBlobsPerBlock) { Transaction? bestTxLight = GetFirsts().Min; @@ -85,7 +85,7 @@ public override IEnumerable GetBlobTransactions() break; } - if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + fullBlobTx.BlobVersionedHashes!.Length <= MaxNumberOfBlobsInBlock) + if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + fullBlobTx.BlobVersionedHashes!.Length <= Eip4844Constants.MaxBlobsPerBlock) { yield return fullBlobTx!; pickedBlobs += fullBlobTx.BlobVersionedHashes.Length; @@ -93,7 +93,7 @@ public override IEnumerable GetBlobTransactions() if (TryRemove(bestTxLight.Hash)) { - blobTxsToReadd ??= new(MaxNumberOfBlobsInBlock); + blobTxsToReadd ??= new(Eip4844Constants.MaxBlobsPerBlock); blobTxsToReadd.Add(fullBlobTx!); } } From 5fbf208cf7170c7393be81c467bf8134f1e968c2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 15 Aug 2023 14:25:21 +0200 Subject: [PATCH 069/148] whitespace --- .../Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index c55a9f44d22..c410952cdfe 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -261,7 +261,7 @@ private IParityRpcModule CreateParityRpcModule(IPeerManager? peerManager = null) _receiptStorage, new Enode(TestItem.PublicKeyA, IPAddress.Loopback, 8545), _signerStore, - new MemKeyStore(new[] { TestItem.PrivateKeyA },Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), + new MemKeyStore(new[] { TestItem.PrivateKeyA }, Path.Combine("testKeyStoreDir", Path.GetRandomFileName())), MainnetSpecProvider.Instance, peerManager ?? Substitute.For()); } From 93d00b1114fc68d9748f0fbb69a80bb81d554ec0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 16 Aug 2023 18:39:24 +0200 Subject: [PATCH 070/148] cosmetic in BlobTxStorage + add tests --- .../BlobTxStorageTests.cs | 36 +++++++++++++++++++ .../Nethermind.TxPool/BlobTxStorage.cs | 2 +- 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs diff --git a/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs new file mode 100644 index 00000000000..e28818fe021 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using FluentAssertions; +using Nethermind.Core; +using Nethermind.Core.Test.Builders; +using Nethermind.Db; +using NUnit.Framework; + +namespace Nethermind.TxPool.Test; + +[TestFixture] +public class BlobTxStorageTests +{ + [Test] + public void should_throw_when_trying_to_add_null_tx() + { + BlobTxStorage blobTxStorage = new(new MemDb()); + + Action act = () => blobTxStorage.Add(null); + act.Should().Throw(); + } + + [Test] + public void should_throw_when_trying_to_add_tx_with_null_hash() + { + BlobTxStorage blobTxStorage = new(new MemDb()); + + Transaction tx = Build.A.Transaction.TestObject; + tx.Hash = null; + + Action act = () => blobTxStorage.Add(tx); + act.Should().Throw(); + } +} diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 2896aa6b5a2..9ff528bb79b 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -55,7 +55,7 @@ public IEnumerable GetAll() public void Add(Transaction transaction) { - if (transaction == null || transaction.Hash == null) + if (transaction?.Hash is null) { throw new ArgumentNullException(nameof(transaction)); } From b8030ce8937cba52ca7e7a2135ad8c3c4bcec034 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 16 Aug 2023 20:25:55 +0200 Subject: [PATCH 071/148] make tx comparisons more readable --- .../Comparison/ByHashTxComparer.cs | 12 +++--- .../Comparison/CompareReplacedBlobTx.cs | 22 +++++----- .../Comparison/CompareReplacedTxByFee.cs | 42 ++++++++++--------- .../Comparison/CompareTxByGasLimit.cs | 10 ++--- .../Comparison/CompareTxByNonce.cs | 10 ++--- .../Comparison/CompareTxByPoolIndex.cs | 10 ++--- .../Comparison/CompareTxByTimestamp.cs | 10 ++--- .../CompetingTransactionEqualityComparer.cs | 6 +-- .../Comparison/TxComparisonResult.cs | 11 +++++ 9 files changed, 74 insertions(+), 59 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/ByHashTxComparer.cs b/src/Nethermind/Nethermind.TxPool/Comparison/ByHashTxComparer.cs index c7ef35ed43f..f972b8f813b 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/ByHashTxComparer.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/ByHashTxComparer.cs @@ -15,16 +15,16 @@ public class ByHashTxComparer : IComparer, IEqualityComparer Compare(x, y) == 0; + public bool Equals(Transaction? x, Transaction? y) => Compare(x, y) == TxComparisonResult.NotDecided; public int GetHashCode(Transaction obj) => obj.Hash?.GetHashCode() ?? 0; } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index 243543864ba..1646e3f37c5 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -17,24 +17,24 @@ private CompareReplacedBlobTx() { } // To replace old blob transaction, new transaction needs to have fee at least 2x higher than current fee. // 2x higher must be MaxPriorityFeePerGas, MaxFeePerGas and MaxFeePerDataGas - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; // do not allow to replace blob tx by the one with lower number of blobs - if (y.BlobVersionedHashes is null || x.BlobVersionedHashes is null) return 1; - if (y.BlobVersionedHashes.Length > x.BlobVersionedHashes.Length) return 1; + if (oldTx.BlobVersionedHashes is null || newTx.BlobVersionedHashes is null) return TxComparisonResult.KeepOld; + if (oldTx.BlobVersionedHashes.Length > newTx.BlobVersionedHashes.Length) return TxComparisonResult.KeepOld; // always allow replacement of zero fee txs - if (y.MaxFeePerGas.IsZero) return -1; //ToDo: do we need it? + if (oldTx.MaxFeePerGas.IsZero) return TxComparisonResult.TakeNew; //ToDo: do we need it? - if (y.MaxFeePerGas * 2 > x.MaxFeePerGas) return 1; - if (y.MaxPriorityFeePerGas * 2 > x.MaxPriorityFeePerGas) return 1; - if (y.MaxFeePerBlobGas * 2 > x.MaxFeePerBlobGas) return 1; + if (oldTx.MaxFeePerGas * 2 > newTx.MaxFeePerGas) return TxComparisonResult.KeepOld; + if (oldTx.MaxPriorityFeePerGas * 2 > newTx.MaxPriorityFeePerGas) return TxComparisonResult.KeepOld; + if (oldTx.MaxFeePerBlobGas * 2 > newTx.MaxFeePerBlobGas) return TxComparisonResult.KeepOld; // if we are here, it means that all new fees are at least 2x higher than old ones, so replacement is allowed - return -1; + return TxComparisonResult.TakeNew; } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedTxByFee.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedTxByFee.cs index 1982ac4574e..f44bcf92462 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedTxByFee.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedTxByFee.cs @@ -20,34 +20,38 @@ private CompareReplacedTxByFee() { } // It is required to avoid acceptance and propagation of transaction with almost the same fee as replaced one. private const int PartOfFeeRequiredToIncrease = 10; - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; // always allow replacement of zero fee txs (in legacy txs MaxFeePerGas equals GasPrice) - if (y.MaxFeePerGas.IsZero) return -1; + if (oldTx.MaxFeePerGas.IsZero) return TxComparisonResult.TakeNew; - if (!x.Supports1559 && !y.Supports1559) + if (!newTx.Supports1559 && !oldTx.Supports1559) { - y.GasPrice.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpGasPrice); - int gasPriceResult = (y.GasPrice + bumpGasPrice).CompareTo(x.GasPrice); - // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease - // never return 0 - it's allowed or not - return gasPriceResult != 0 ? gasPriceResult : bumpGasPrice > 0 ? -1 : 1; + oldTx.GasPrice.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpGasPrice); + int gasPriceResult = (oldTx.GasPrice + bumpGasPrice).CompareTo(newTx.GasPrice); + // return TakeNew if fee bump is exactly by PartOfFeeRequiredToIncrease + // never return NotDecided - it's allowed or not + return gasPriceResult != 0 ? gasPriceResult : bumpGasPrice > 0 + ? TxComparisonResult.TakeNew + : TxComparisonResult.KeepOld; } /* MaxFeePerGas for legacy will be GasPrice and MaxPriorityFeePerGas will be GasPrice too so we can compare legacy txs without any problems */ - y.MaxFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxFeePerGas); - if (y.MaxFeePerGas + bumpMaxFeePerGas > x.MaxFeePerGas) return 1; - - y.MaxPriorityFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxPriorityFeePerGas); - int result = (y.MaxPriorityFeePerGas + bumpMaxPriorityFeePerGas).CompareTo(x.MaxPriorityFeePerGas); - // return -1 (replacement accepted) if fee bump is exactly by PartOfFeeRequiredToIncrease - // never return 0 - it's allowed or not - return result != 0 ? result : (bumpMaxFeePerGas > 0 && bumpMaxPriorityFeePerGas > 0) ? -1 : 1; + oldTx.MaxFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxFeePerGas); + if (oldTx.MaxFeePerGas + bumpMaxFeePerGas > newTx.MaxFeePerGas) return TxComparisonResult.KeepOld; + + oldTx.MaxPriorityFeePerGas.Divide(PartOfFeeRequiredToIncrease, out UInt256 bumpMaxPriorityFeePerGas); + int result = (oldTx.MaxPriorityFeePerGas + bumpMaxPriorityFeePerGas).CompareTo(newTx.MaxPriorityFeePerGas); + // return TakeNew if fee bump is exactly by PartOfFeeRequiredToIncrease + // never return NotDecided - it's allowed or not + return result != 0 ? result : (bumpMaxFeePerGas > 0 && bumpMaxPriorityFeePerGas > 0) + ? TxComparisonResult.TakeNew + : TxComparisonResult.KeepOld; } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByGasLimit.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByGasLimit.cs index 9d8a3949546..5d06e4dde03 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByGasLimit.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByGasLimit.cs @@ -15,14 +15,14 @@ public class CompareTxByGasLimit : IComparer private CompareTxByGasLimit() { } - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; // then by gas limit ascending - return x.GasLimit.CompareTo(y.GasLimit); + return newTx.GasLimit.CompareTo(oldTx.GasLimit); } } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByNonce.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByNonce.cs index e2785b450a6..fd2b7228df5 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByNonce.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByNonce.cs @@ -15,14 +15,14 @@ public class CompareTxByNonce : IComparer private CompareTxByNonce() { } - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; // compare by nonce ascending - return x.Nonce.CompareTo(y.Nonce); + return newTx.Nonce.CompareTo(oldTx.Nonce); } } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByPoolIndex.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByPoolIndex.cs index 313e85c73ec..d9874d327e6 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByPoolIndex.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByPoolIndex.cs @@ -15,13 +15,13 @@ public class CompareTxByPoolIndex : IComparer private CompareTxByPoolIndex() { } - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; - return x.PoolIndex.CompareTo(y.PoolIndex); + return newTx.PoolIndex.CompareTo(oldTx.PoolIndex); } } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByTimestamp.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByTimestamp.cs index 9ba9166398c..12cd4ae309a 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByTimestamp.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareTxByTimestamp.cs @@ -15,13 +15,13 @@ public class CompareTxByTimestamp : IComparer private CompareTxByTimestamp() { } - public int Compare(Transaction? x, Transaction? y) + public int Compare(Transaction? newTx, Transaction? oldTx) { - if (ReferenceEquals(x, y)) return 0; - if (ReferenceEquals(null, y)) return 1; - if (ReferenceEquals(null, x)) return -1; + if (ReferenceEquals(newTx, oldTx)) return TxComparisonResult.NotDecided; + if (ReferenceEquals(null, oldTx)) return TxComparisonResult.KeepOld; + if (ReferenceEquals(null, newTx)) return TxComparisonResult.TakeNew; - return x.Timestamp.CompareTo(y.Timestamp); + return newTx.Timestamp.CompareTo(oldTx.Timestamp); } } } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompetingTransactionEqualityComparer.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompetingTransactionEqualityComparer.cs index 5f9c5289907..954418e802a 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompetingTransactionEqualityComparer.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompetingTransactionEqualityComparer.cs @@ -9,7 +9,7 @@ namespace Nethermind.TxPool.Comparison { /// /// Comparer to check if two pending s compete with each other. - /// s compete with each other if they have same and . In that case only one transaction can go into chain. + /// s compete with each other if they have same and . In that case only one transaction can go into chain. /// public class CompetingTransactionEqualityComparer : IEqualityComparer { @@ -17,8 +17,8 @@ public class CompetingTransactionEqualityComparer : IEqualityComparer - ReferenceEquals(x, y) || !ReferenceEquals(x, null) && !ReferenceEquals(y, null) && x.SenderAddress == y.SenderAddress && x.Nonce == y.Nonce; + public bool Equals(Transaction? newTx, Transaction? oldTx) => + ReferenceEquals(newTx, oldTx) || !ReferenceEquals(newTx, null) && !ReferenceEquals(oldTx, null) && newTx.SenderAddress == oldTx.SenderAddress && newTx.Nonce == oldTx.Nonce; public int GetHashCode(Transaction? obj) => HashCode.Combine(obj?.SenderAddress, obj?.Nonce); } diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs b/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs new file mode 100644 index 00000000000..f4334a3f989 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs @@ -0,0 +1,11 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.TxPool.Comparison; + +public readonly struct TxComparisonResult +{ + public const int NotDecided = 0; + public const int KeepOld = 1; + public const int TakeNew = -1; +} From b1b2c1509a277d5069f9cfeda43274aa444745af Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 12:55:49 +0200 Subject: [PATCH 072/148] cosmetics in TxTypeTxFilter --- src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs index cc103fe2a25..075503b96cc 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs @@ -22,8 +22,8 @@ public TxTypeTxFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) { - if ((tx.SupportsBlobs ? _txs : _blobTxs) - .ContainsBucket(tx.SenderAddress!)) // as unknownSenderFilter will run before this one + TxDistinctSortedPool otherTxTypePool = (tx.SupportsBlobs ? _txs : _blobTxs); + if (otherTxTypePool.ContainsBucket(tx.SenderAddress!)) // as unknownSenderFilter will run before this one { Metrics.PendingTransactionsConflictingTxType++; return AcceptTxResult.PendingTxsOfOtherType; From fc8e258ca38fe73f3529856189f480379cb023ab Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 13:31:42 +0200 Subject: [PATCH 073/148] filter blob txs in FeeTooLowFilter just as other types --- .../Nethermind.TxPool/Filters/FeeTooLowFilter.cs | 9 ++++++--- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs index 6c15155b11b..49afaa854e2 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs @@ -19,14 +19,16 @@ internal sealed class FeeTooLowFilter : IIncomingTxFilter private readonly IChainHeadSpecProvider _specProvider; private readonly IChainHeadInfoProvider _headInfo; private readonly TxDistinctSortedPool _txs; + private readonly TxDistinctSortedPool _blobTxs; private readonly bool _thereIsPriorityContract; private readonly ILogger _logger; - public FeeTooLowFilter(IChainHeadInfoProvider headInfo, TxDistinctSortedPool txs, bool thereIsPriorityContract, ILogger logger) + public FeeTooLowFilter(IChainHeadInfoProvider headInfo, TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs, bool thereIsPriorityContract, ILogger logger) { _specProvider = headInfo.SpecProvider; _headInfo = headInfo; _txs = txs; + _blobTxs = blobTxs; _thereIsPriorityContract = thereIsPriorityContract; _logger = logger; } @@ -41,7 +43,7 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO } bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; - if (isLocal || tx.SupportsBlobs) + if (isLocal) { return AcceptTxResult.Accepted; } @@ -59,7 +61,8 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO AcceptTxResult.FeeTooLow.WithMessage("Affordable gas price is 0"); } - if (_txs.IsFull() && _txs.TryGetLast(out Transaction? lastTx) + TxDistinctSortedPool relevantPool = (tx.SupportsBlobs ? _blobTxs : _txs); + if (relevantPool.IsFull() && relevantPool.TryGetLast(out Transaction? lastTx) && affordableGasPrice <= lastTx?.GasBottleneck) { Metrics.PendingTransactionsTooLowFee++; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index c00079081d9..3b89b1e6276 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -109,7 +109,7 @@ public TxPool(IEthereumEcdsa ecdsa, _preHashFilters = new IIncomingTxFilter[] { new GasLimitTxFilter(_headInfo, txPoolConfig, _logger), - new FeeTooLowFilter(_headInfo, _transactions, thereIsPriorityContract, _logger), + new FeeTooLowFilter(_headInfo, _transactions, _blobTransactions, thereIsPriorityContract, _logger), new MalformedTxFilter(_specProvider, validator, _logger) }; From a0bd04bc78e25aaf7b8319103278c16df7172d80 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 13:31:59 +0200 Subject: [PATCH 074/148] adjust test --- .../TxPoolTests.Blobs.cs | 39 ++++++++++--------- 1 file changed, 20 insertions(+), 19 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 03f0c922504..d99423e00ee 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; @@ -16,46 +17,46 @@ namespace Nethermind.TxPool.Test [TestFixture] public partial class TxPoolTests { - // blob collection is designed to be infinite, so there is no point in checking if is full and comparing - // incoming tx with the worst already pending one (candidate for evicting). There will be no eviction. [Test] - public void should_reject_tx_with_FeeTooLow_only_if_is_not_blob_type() + public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, false)] bool isBlob, [Values(true, false)] bool persistentStorageEnabled) { - TxPoolConfig txPoolConfig = new() { Size = 10 }; + const int poolSize = 10; + TxPoolConfig txPoolConfig = new() + { + Size = isBlob ? 0 : poolSize, + PersistentBlobStorageEnabled = persistentStorageEnabled, + PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, + InMemoryBlobPoolSize = persistentStorageEnabled ? 0 : poolSize + }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); EnsureSenderBalance(TestItem.AddressB, UInt256.MaxValue); - for (int i = 0; i < txPoolConfig.Size; i++) + for (int i = 0; i < poolSize; i++) { Transaction tx = Build.A.Transaction .WithNonce((UInt256)i) - .WithType(TxType.EIP1559) + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(1.GWei() + (UInt256)(100 - i)) .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)(100 - i)) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); } - _txPool.GetPendingTransactionsCount().Should().Be(txPoolConfig.Size); + _txPool.GetPendingTransactionsCount().Should().Be(isBlob ? 0 : poolSize); + _txPool.GetPendingBlobTransactionsCount().Should().Be(isBlob ? poolSize : 0); - Transaction feeTooLow1559Tx = Build.A.Transaction - .WithType(TxType.EIP1559) - .WithMaxFeePerGas(1.GWei() + UInt256.One) - .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) + Transaction feeTooLowTx = Build.A.Transaction .WithNonce(UInt256.Zero) - .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; - - Transaction feeTooLowBlobTx = Build.A.Transaction - .WithShardBlobTxTypeAndFields() + .WithType(isBlob ? TxType.Blob : TxType.EIP1559) + .WithShardBlobTxTypeAndFieldsIfBlobTx() .WithMaxFeePerGas(1.GWei() + UInt256.One) .WithMaxPriorityFeePerGas(1.GWei() + UInt256.One) - .WithNonce(UInt256.Zero) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyB).TestObject; - - _txPool.SubmitTx(feeTooLow1559Tx, TxHandlingOptions.None).Should().Be(AcceptTxResult.FeeTooLow); - _txPool.SubmitTx(feeTooLowBlobTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); + _txPool.SubmitTx(feeTooLowTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.FeeTooLow); } [Test] From ff01c02959cad02d930faa5d2f4ddeaef9d65a20 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 13:58:50 +0200 Subject: [PATCH 075/148] fix whitespaces and file encodings --- src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs | 2 +- .../Nethermind.TxPool/Comparison/TxComparisonResult.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs index e28818fe021..fee8743c712 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs b/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs index f4334a3f989..84896e596fa 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/TxComparisonResult.cs @@ -1,9 +1,9 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only namespace Nethermind.TxPool.Comparison; -public readonly struct TxComparisonResult +public readonly struct TxComparisonResult { public const int NotDecided = 0; public const int KeepOld = 1; From 40456ecf3ea042e3eb197342368d4b358d53b6fc Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 14:17:33 +0200 Subject: [PATCH 076/148] simplification in TxPool --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 3b89b1e6276..4180dcf644d 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -580,11 +580,8 @@ public bool RemoveTransaction(Keccak? hash) bool hasBeenRemoved; lock (_locker) { - hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction); - if (!hasBeenRemoved) - { - hasBeenRemoved = _blobTransactions.TryRemove(hash, out transaction); - } + hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction) + ||_blobTransactions.TryRemove(hash, out transaction); if (transaction is null || !hasBeenRemoved) return false; From 58e8b9696d5d883591b272116fe2122ba9bc1d4d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 14:29:40 +0200 Subject: [PATCH 077/148] another whitespace fix --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 4180dcf644d..ed1f6d53140 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -581,7 +581,7 @@ public bool RemoveTransaction(Keccak? hash) lock (_locker) { hasBeenRemoved = _transactions.TryRemove(hash, out Transaction? transaction) - ||_blobTransactions.TryRemove(hash, out transaction); + || _blobTransactions.TryRemove(hash, out transaction); if (transaction is null || !hasBeenRemoved) return false; From a026341043c86df6dd33a0b7eaa024a250455db6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 15:11:39 +0200 Subject: [PATCH 078/148] override base methods in blob collections instead of creating new ones + simplifications in TxPool --- .../Collections/BlobTxDistinctSortedPool.cs | 6 ------ .../PersistentBlobTxDistinctSortedPool.cs | 5 +++-- .../Nethermind.TxPool/Collections/SortedPool.cs | 4 ++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 14 +++++++------- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 41b0bef4f59..5befe8e981f 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -18,12 +18,6 @@ public BlobTxDistinctSortedPool(int capacity, IComparer comparer, I protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - public virtual bool TryInsertBlobTx(Transaction fullBlobTx, out Transaction? removed) - => base.TryInsert(fullBlobTx.Hash, fullBlobTx, out removed); - - public virtual bool TryGetBlobTx(ValueKeccak hash, out Transaction? fullBlobTx) - => base.TryGetValue(hash, out fullBlobTx); - public virtual IEnumerable GetBlobTransactions() { int pickedBlobs = 0; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index af042f0ed73..4ab8890478e 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -38,7 +39,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) } } - public override bool TryInsertBlobTx(Transaction fullBlobTx, out Transaction? removed) + public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Transaction? removed) { if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out removed)) { @@ -50,7 +51,7 @@ public override bool TryInsertBlobTx(Transaction fullBlobTx, out Transaction? re return false; } - public override bool TryGetBlobTx(ValueKeccak hash, out Transaction? fullBlobTx) + public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Transaction? fullBlobTx) { if (base.ContainsValue(hash)) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 5acd3b00679..a3ef472ad8b 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -273,7 +273,7 @@ protected bool ContainsValue(TKey key) /// Returned element or null. /// If element retrieval succeeded. True if element was present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) + public virtual bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) { return _cacheMap.TryGetValue(key, out value) && value != null; } @@ -286,7 +286,7 @@ public bool TryGetValue(TKey key, [NotNullWhen(true)] out TValue? value) /// Element removed because of exceeding capacity /// If element was inserted. False if element was already present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public bool TryInsert(TKey key, TValue value, out TValue? removed) + public virtual bool TryInsert(TKey key, TValue value, out TValue? removed) { if (CanInsert(key, value)) { diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ed1f6d53140..1cf45119d01 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -376,15 +376,14 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe { bool eip1559Enabled = _specProvider.GetCurrentHeadSpec().IsEip1559Enabled; UInt256 effectiveGasPrice = tx.CalculateEffectiveGasPrice(eip1559Enabled, _headInfo.CurrentBaseFee); + TxDistinctSortedPool relevantPool = (tx.SupportsBlobs ? _blobTransactions : _transactions); - (tx.SupportsBlobs ? _blobTransactions : _transactions).TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); + relevantPool.TryGetBucketsWorstValue(tx.SenderAddress!, out Transaction? worstTx); tx.GasBottleneck = (worstTx is null || effectiveGasPrice <= worstTx.GasBottleneck) ? effectiveGasPrice : worstTx.GasBottleneck; - bool inserted = (tx.SupportsBlobs - ? _blobTransactions.TryInsertBlobTx(tx, out Transaction? removed) - : _transactions.TryInsert(tx.Hash!, tx, out removed)); + bool inserted = relevantPool.TryInsert(tx.Hash!, tx, out Transaction? removed); if (!inserted) { @@ -403,7 +402,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe return AcceptTxResult.FeeTooLowToCompete; } - (tx.SupportsBlobs ? _blobTransactions : _transactions).UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); + relevantPool.UpdateGroup(tx.SenderAddress!, state.SenderAccount, UpdateBucketWithAddedTransaction); Metrics.PendingTransactionsAdded++; if (tx.Supports1559) { Metrics.Pending1559TransactionsAdded++; } if (tx.SupportsBlobs) { Metrics.PendingBlobTransactionsAdded++; } @@ -603,7 +602,7 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) lock (_locker) { return _transactions.TryGetValue(hash, out transaction) - || _blobTransactions.TryGetBlobTx(hash, out transaction) + || _blobTransactions.TryGetValue(hash, out transaction) || _broadcaster.TryGetPersistentTx(hash, out transaction); } } @@ -625,8 +624,9 @@ public UInt256 GetLatestPendingNonce(Address address) return maxPendingNonce; } + TxDistinctSortedPool relevantPool = (hasPendingTxs ? _transactions : _blobTransactions); // we are not doing any updating, but lets just use a thread-safe method without any data copying like snapshot - (hasPendingTxs ? _transactions : _blobTransactions).UpdateGroup(address, (_, transactions) => + relevantPool.UpdateGroup(address, (_, transactions) => { // This is under the assumption that the addressTransactions are sorted by Nonce. if (transactions.Count > 0) From bb6a6b4d1156dc4c6c63b884aef7264cb711770a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 15:41:45 +0200 Subject: [PATCH 079/148] cosmetics --- src/Nethermind/Nethermind.Core/Eip4844Constants.cs | 2 +- src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Nethermind/Nethermind.Core/Eip4844Constants.cs b/src/Nethermind/Nethermind.Core/Eip4844Constants.cs index fbd82dd753d..ab502e4ebc4 100644 --- a/src/Nethermind/Nethermind.Core/Eip4844Constants.cs +++ b/src/Nethermind/Nethermind.Core/Eip4844Constants.cs @@ -8,8 +8,8 @@ namespace Nethermind.Core; public class Eip4844Constants { public const int MinBlobsPerTransaction = 1; - public const int MaxBlobsPerBlock = 6; + public const ulong BlobGasPerBlob = 1 << 17; public const ulong MaxBlobGasPerBlock = BlobGasPerBlob * MaxBlobsPerBlock; public const ulong TargetBlobGasPerBlock = MaxBlobGasPerBlock / 2; diff --git a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs index f6a46565050..9336ae51a56 100644 --- a/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs +++ b/src/Nethermind/Nethermind.TxPool/TransactionExtensions.cs @@ -81,9 +81,9 @@ internal static bool IsOverflowWhenAddingTxCostToCumulative(this Transaction tx, if (tx.SupportsBlobs) { // if tx.SupportsBlobs and has BlobVersionedHashes = null, it will throw on earlier step of validation, in TxValidator - overflow |= UInt256.MultiplyOverflow(Eip4844Constants.BlobGasPerBlob, (UInt256)tx.BlobVersionedHashes!.Length, out UInt256 dataGas); - overflow |= UInt256.MultiplyOverflow(dataGas, tx.MaxFeePerBlobGas ?? UInt256.MaxValue, out UInt256 dataGasCost); - overflow |= UInt256.AddOverflow(cumulativeCost, dataGasCost, out cumulativeCost); + overflow |= UInt256.MultiplyOverflow(Eip4844Constants.BlobGasPerBlob, (UInt256)tx.BlobVersionedHashes!.Length, out UInt256 blobGas); + overflow |= UInt256.MultiplyOverflow(blobGas, tx.MaxFeePerBlobGas ?? UInt256.MaxValue, out UInt256 blobGasCost); + overflow |= UInt256.AddOverflow(cumulativeCost, blobGasCost, out cumulativeCost); } return overflow; From 768ba88f4872c4e887afa2e0d5aa7c1d530642b6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 17:01:03 +0200 Subject: [PATCH 080/148] add BlobTxStorage parameterless constructor that uses MemDb and simplify tests --- src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs | 2 +- .../Nethermind.Clique.Test/CliqueBlockProducerTests.cs | 2 +- .../Nethermind.Core.Test/Blockchain/TestBlockchain.cs | 2 +- .../Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs | 2 +- .../Nethermind.Synchronization.Test/SyncThreadTests.cs | 2 +- src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs | 4 ++-- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 6 +++--- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 5 +++++ 9 files changed, 16 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs index d1627fd9ced..42fe07c5b1a 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/ReorgTests.cs @@ -61,7 +61,7 @@ public void Setup() LimboLogs.Instance); TxPool.TxPool txPool = new( ecdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(specProvider, _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs index 643cb9b670b..62d0b824deb 100644 --- a/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs +++ b/src/Nethermind/Nethermind.Clique.Test/CliqueBlockProducerTests.cs @@ -108,7 +108,7 @@ public On CreateNode(PrivateKey privateKey, bool withGenesisAlreadyProcessed = f new TransactionComparerProvider(specProvider, blockTree); TxPool.TxPool txPool = new(_ethereumEcdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(GoerliSpecProvider.Instance), blockTree, stateProvider), new TxPoolConfig(), new TxValidator(goerliSpecProvider.ChainId), diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index 015e1b2df4d..c2e67b3c8a2 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -265,7 +265,7 @@ protected virtual IBlockProducer CreateTestBlockProducer(TxPoolTxSource txPoolTx protected virtual TxPool.TxPool CreateTxPool() => new( EthereumEcdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), new TxPoolConfig(), new TxValidator(SpecProvider.ChainId), diff --git a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs index c410952cdfe..3cce198ad84 100644 --- a/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs +++ b/src/Nethermind/Nethermind.JsonRpc.Test/Modules/ParityRpcModuleTests.cs @@ -74,7 +74,7 @@ public void Initialize() _blockTree = new BlockTree(blockDb, headerDb, blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, logger); _txPool = new TxPool.TxPool(_ethereumEcdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(specProvider), _blockTree, stateProvider), new TxPoolConfig(), new TxValidator(specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs index 9a751668e84..212687e7306 100644 --- a/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs +++ b/src/Nethermind/Nethermind.Synchronization.Test/SyncThreadTests.cs @@ -272,7 +272,7 @@ private SyncTestContext CreateSyncManager(int index) new TransactionComparerProvider(specProvider, tree); TxPool.TxPool txPool = new(ecdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(specProvider, tree, stateReader), new TxPoolConfig(), new TxValidator(specProvider.ChainId), diff --git a/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs index fee8743c712..385ae8e9d9d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/BlobTxStorageTests.cs @@ -16,7 +16,7 @@ public class BlobTxStorageTests [Test] public void should_throw_when_trying_to_add_null_tx() { - BlobTxStorage blobTxStorage = new(new MemDb()); + BlobTxStorage blobTxStorage = new(); Action act = () => blobTxStorage.Add(null); act.Should().Throw(); @@ -25,7 +25,7 @@ public void should_throw_when_trying_to_add_null_tx() [Test] public void should_throw_when_trying_to_add_tx_with_null_hash() { - BlobTxStorage blobTxStorage = new(new MemDb()); + BlobTxStorage blobTxStorage = new(); Transaction tx = Build.A.Transaction.TestObject; tx.Hash = null; diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index d99423e00ee..35f52d9a7dc 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -63,7 +63,7 @@ public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, f public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) { TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; - BlobTxStorage blobTxStorage = new(new MemDb()); + BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -93,7 +93,7 @@ public void should_add_blob_tx_and_return_when_requested([Values(true, false)] b public void should_not_throw_when_asking_for_non_existing_tx() { TxPoolConfig txPoolConfig = new() { Size = 10 }; - BlobTxStorage blobTxStorage = new(new MemDb()); + BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); @@ -176,7 +176,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() { TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = true }; - BlobTxStorage blobTxStorage = new(new MemDb()); + BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index 2011ea1ada0..c2cf56efe07 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1661,7 +1661,7 @@ private TxPool CreatePool( specProvider ??= MainnetSpecProvider.Instance; ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, _blockTree); - txStorage ??= new BlobTxStorage(new MemDb()); + txStorage ??= new BlobTxStorage(); _headInfo = chainHeadInfoProvider; _headInfo ??= new ChainHeadInfoProvider(specProvider, _blockTree, _stateProvider); diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 9ff528bb79b..6312c263333 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -15,6 +15,11 @@ public class BlobTxStorage : ITxStorage { private readonly IDb _database; + public BlobTxStorage() + { + _database = new MemDb(); + } + public BlobTxStorage(IDb database) { _database = database ?? throw new ArgumentNullException(nameof(database)); From 29010687b17d0119a35a48bbab9a9dc0e37469fa Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 17 Aug 2023 18:48:13 +0200 Subject: [PATCH 081/148] remove unused things --- .../Ethereum.Test.Base/BlockchainTestBase.cs | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs index f8c4edc4a4f..ed6974678dc 100644 --- a/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs +++ b/src/Nethermind/Ethereum.Test.Base/BlockchainTestBase.cs @@ -130,17 +130,7 @@ protected async Task RunTest(BlockchainTest test, Stopwatch? IWorldState stateProvider = new WorldState(trieStore, codeDb, _logManager); MemDb blockInfoDb = new MemDb(); IBlockTree blockTree = new BlockTree(new MemDb(), new MemDb(), blockInfoDb, new ChainLevelInfoRepository(blockInfoDb), specProvider, NullBloomStorage.Instance, _logManager); - ITransactionComparerProvider transactionComparerProvider = new TransactionComparerProvider(specProvider, blockTree); IStateReader stateReader = new StateReader(trieStore, codeDb, _logManager); - IChainHeadInfoProvider chainHeadInfoProvider = new ChainHeadInfoProvider(specProvider, blockTree, stateReader); - ITxStorage txStorage = new BlobTxStorage(new MemDb()); - ITxPool transactionPool = new TxPool(ecdsa, - txStorage, - chainHeadInfoProvider, - new TxPoolConfig(), - new TxValidator(specProvider.ChainId), - _logManager, - transactionComparerProvider.GetDefaultComparer()); IReceiptStorage receiptStorage = NullReceiptStorage.Instance; IBlockhashProvider blockhashProvider = new BlockhashProvider(blockTree, _logManager); From a35b8af211f6ad5a3bf012c4d9c1907de0576773 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 21 Aug 2023 12:09:23 +0200 Subject: [PATCH 082/148] post-merge fix --- .../Nethermind.TxPool.Test/TxBroadcasterTests.cs | 2 +- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index dcdf8f65fa6..bf96b09da02 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -179,7 +179,7 @@ public void should_announce_details_of_full_blob_tx() Transaction lightTx = new LightTransaction(tx); int size = tx.GetLength(); - size.Should().Be(131324); + size.Should().Be(131320); lightTx.GetLength().Should().Be(size); _broadcaster.Broadcast(tx, true); diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 35f52d9a7dc..8ec74b9faeb 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -379,12 +379,12 @@ Transaction GetTx(bool isBlob, UInt256 nonce) } [TestCase(0, 97)] - [TestCase(1, 131324)] - [TestCase(2, 262534)] - [TestCase(3, 393741)] - [TestCase(4, 524947)] - [TestCase(5, 656156)] - [TestCase(6, 787365)] + [TestCase(1, 131320)] + [TestCase(2, 262530)] + [TestCase(3, 393737)] + [TestCase(4, 524943)] + [TestCase(5, 656152)] + [TestCase(6, 787361)] public void should_calculate_size_of_blob_tx_correctly(int numberOfBlobs, int expectedLength) { Transaction blobTx = Build.A.Transaction From 6b4a4382c7294683f643625d859664c6fedc8889 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 22 Aug 2023 12:27:41 +0200 Subject: [PATCH 083/148] move capacity insurances to inner classes --- .../Collections/BlobTxDistinctSortedPool.cs | 12 ++++++++++++ .../PersistentBlobTxDistinctSortedPool.cs | 10 +++++++++- .../Collections/TxDistinctSortedPool.cs | 11 +++++++++++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 15 ++------------- 4 files changed, 34 insertions(+), 14 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 5befe8e981f..4e809847f10 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -1,6 +1,7 @@ // SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only +using System; using System.Collections.Generic; using Nethermind.Core; using Nethermind.Core.Crypto; @@ -10,9 +11,12 @@ namespace Nethermind.TxPool.Collections; public class BlobTxDistinctSortedPool : TxDistinctSortedPool { + private readonly ILogger _logger; + public BlobTxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, logManager) { + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); } protected override IComparer GetReplacementComparer(IComparer comparer) @@ -54,6 +58,14 @@ public virtual IEnumerable GetBlobTransactions() } } + public override void EnsureCapacity() + { + base.EnsureCapacity(); + + if (Count > _poolCapacity && _logger.IsWarn) + _logger.Warn($"Blob TxPool exceeds the config size {Count}/{_poolCapacity}"); + } + /// /// For tests only - to test sorting /// diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 4ab8890478e..c538d461448 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -22,7 +22,7 @@ public PersistentBlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfi { _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _blobTxCache = new(txPoolConfig.BlobCacheSize, txPoolConfig.BlobCacheSize, "blob txs cache"); - _logger = logManager.GetClassLogger(); + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); RecreateLightTxCollectionAndCache(blobTxStorage); } @@ -116,4 +116,12 @@ protected override bool Remove(ValueKeccak hash, Transaction tx) _blobTxStorage.Delete(hash); return base.Remove(hash, tx); } + + public override void EnsureCapacity() + { + base.EnsureCapacity(); + + if (_logger.IsDebug && Count == _poolCapacity) + _logger.Debug($"Blob persistent storage has reached max size of {_poolCapacity}, blob txs can be evicted now"); + } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs index 9326ceb3ec4..6647bc0858a 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs @@ -19,10 +19,15 @@ namespace Nethermind.TxPool.Collections public class TxDistinctSortedPool : DistinctValueSortedPool { private readonly List _transactionsToRemove = new(); + protected int _poolCapacity; + private readonly ILogger _logger; + public TxDistinctSortedPool(int capacity, IComparer comparer, ILogManager logManager) : base(capacity, comparer, CompetingTransactionEqualityComparer.Instance, logManager) { + _poolCapacity = capacity; + _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); } protected override IComparer GetUniqueComparer(IComparer comparer) => comparer.GetPoolUniqueTxComparer(); @@ -124,5 +129,11 @@ public void UpdateGroup(Address groupKey, Account groupValue, Func _poolCapacity && _logger.IsWarn) + _logger.Warn($"TxPool exceeds the config size {Count}/{_poolCapacity}"); + } } } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 1cf45119d01..b401bc0e9fb 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -497,21 +497,10 @@ private void UpdateBuckets() { lock (_locker) { - // ensure the capacity of the pool - if (_transactions.Count > _txPoolConfig.Size) - if (_logger.IsWarn) _logger.Warn($"TxPool exceeds the config size {_transactions.Count}/{_txPoolConfig.Size}"); + _transactions.EnsureCapacity(); _transactions.UpdatePool(_accounts, _updateBucket); - // ensure the capacity of the blob pool - if (_blobTransactions.Count > (_txPoolConfig.PersistentBlobStorageEnabled - ? _txPoolConfig.PersistentBlobStorageSize - : _txPoolConfig.InMemoryBlobPoolSize)) - if (_logger.IsWarn) _logger.Warn($"Blob TxPool exceeds the config size {_blobTransactions.Count}/{_txPoolConfig.PersistentBlobStorageSize}"); - - if (_txPoolConfig.PersistentBlobStorageEnabled - && _blobTransactions.Count == _txPoolConfig.PersistentBlobStorageSize) - if (_logger.IsDebug) _logger.Debug($"Blob persistent storage has reached max size of {_txPoolConfig.PersistentBlobStorageSize}, blob txs can be evicted now"); - + _blobTransactions.EnsureCapacity(); _blobTransactions.UpdatePool(_accounts, _updateBucket); } } From 2ff8309f07d7c6297b7733e7f2b622e8b169243c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 22 Aug 2023 12:28:23 +0200 Subject: [PATCH 084/148] add test for pool capacity --- .../TxPoolTests.Blobs.cs | 29 +++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 8ec74b9faeb..6c0af30f2cb 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -17,6 +17,35 @@ namespace Nethermind.TxPool.Test [TestFixture] public partial class TxPoolTests { + [Test] + public void blob_pool_size_should_be_correct([Values(true, false)] bool persistentStorageEnabled) + { + const int poolSize = 10; + TxPoolConfig txPoolConfig = new() + { + PersistentBlobStorageEnabled = persistentStorageEnabled, + PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, + InMemoryBlobPoolSize = persistentStorageEnabled ? 0 : poolSize + }; + + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + for (int i = 0; i < poolSize; i++) + { + Transaction tx = Build.A.Transaction + .WithNonce((UInt256)i) + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei() + (UInt256)(100 - i)) + .WithMaxPriorityFeePerGas(1.GWei() + (UInt256)(100 - i)) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(AcceptTxResult.Accepted); + } + + _txPool.GetPendingTransactionsCount().Should().Be(0); + _txPool.GetPendingBlobTransactionsCount().Should().Be(poolSize); + } + [Test] public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, false)] bool isBlob, [Values(true, false)] bool persistentStorageEnabled) { From 890ee24655c1c105fc283032275872ba94f6ec65 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 22 Aug 2023 14:09:51 +0200 Subject: [PATCH 085/148] clean --- .../Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs index 1646e3f37c5..122741ed1f2 100644 --- a/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs +++ b/src/Nethermind/Nethermind.TxPool/Comparison/CompareReplacedBlobTx.cs @@ -27,9 +27,6 @@ public int Compare(Transaction? newTx, Transaction? oldTx) if (oldTx.BlobVersionedHashes is null || newTx.BlobVersionedHashes is null) return TxComparisonResult.KeepOld; if (oldTx.BlobVersionedHashes.Length > newTx.BlobVersionedHashes.Length) return TxComparisonResult.KeepOld; - // always allow replacement of zero fee txs - if (oldTx.MaxFeePerGas.IsZero) return TxComparisonResult.TakeNew; //ToDo: do we need it? - if (oldTx.MaxFeePerGas * 2 > newTx.MaxFeePerGas) return TxComparisonResult.KeepOld; if (oldTx.MaxPriorityFeePerGas * 2 > newTx.MaxPriorityFeePerGas) return TxComparisonResult.KeepOld; if (oldTx.MaxFeePerBlobGas * 2 > newTx.MaxFeePerBlobGas) return TxComparisonResult.KeepOld; From 9b36393deb9acafa917bd51221ae8e478c83b984 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 24 Aug 2023 14:25:30 +0200 Subject: [PATCH 086/148] try with synchronization --- .../Collections/BlobTxDistinctSortedPool.cs | 2 ++ .../Collections/PersistentBlobTxDistinctSortedPool.cs | 5 +++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 4e809847f10..8f761689a4a 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Logging; @@ -22,6 +23,7 @@ public BlobTxDistinctSortedPool(int capacity, IComparer comparer, I protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); + [MethodImpl(MethodImplOptions.Synchronized)] public virtual IEnumerable GetBlobTransactions() { int pickedBlobs = 0; diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index c538d461448..ca72bd1762d 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -71,7 +72,7 @@ public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Trans return false; } - // ToDo: add synchronized? + [MethodImpl(MethodImplOptions.Synchronized)] public override IEnumerable GetBlobTransactions() { int pickedBlobs = 0; @@ -94,7 +95,7 @@ public override IEnumerable GetBlobTransactions() if (TryRemove(bestTxLight.Hash)) { - blobTxsToReadd ??= new(Eip4844Constants.MaxBlobsPerBlock); + blobTxsToReadd ??= new(); blobTxsToReadd.Add(fullBlobTx!); } } From 288b0e3604b8b58e7e2a0c5a891cf3fd582722fd Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 25 Aug 2023 13:40:17 +0200 Subject: [PATCH 087/148] refactor block building --- .../TransactionSelectorTests.cs | 23 +++++++++-- .../Producers/TxPoolTxSource.cs | 39 +++++++++++++----- .../Collections/BlobTxDistinctSortedPool.cs | 38 ------------------ .../PersistentBlobTxDistinctSortedPool.cs | 40 ------------------- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 5 ++- .../Nethermind.TxPool/NullTxPool.cs | 15 +++++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 13 +++++- 7 files changed, 73 insertions(+), 100 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index 9cd7c085c7d..72bd8edb5bb 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -23,6 +23,7 @@ using NSubstitute; using NUnit.Framework; using Nethermind.Config; +using Nethermind.Core.Crypto; namespace Nethermind.Blockchain.Test { @@ -237,13 +238,27 @@ void SetAccountStates(IEnumerable
missingAddresses) .ToDictionary( g => g.Key!, g => g.OrderBy(t => t, comparer).ToArray()); - Transaction[] blobTransactions = testCase.Transactions + Dictionary blobTransactions = testCase.Transactions .Where(t => t?.SenderAddress is not null) .Where(t => t.SupportsBlobs) - .OrderBy(t => t, comparer) - .ToArray(); + .GroupBy(t => t.SenderAddress) + .ToDictionary( + g => g.Key!, + g => g.OrderBy(t => t, comparer).ToArray()); transactionPool.GetPendingTransactionsBySender().Returns(transactions); - transactionPool.GetPendingBlobTransactions().Returns(blobTransactions); + transactionPool.GetPendingBlobTransactionsEquivalencesBySender().Returns(blobTransactions); + foreach (KeyValuePair keyValuePair in blobTransactions) + { + foreach (Transaction blobTx in keyValuePair.Value) + { + transactionPool.TryGetPendingBlobTransaction(Arg.Is(h => h == blobTx.Hash), + out Arg.Any()).Returns(x => + { + x[1] = blobTx; + return true; + }); + } + } BlocksConfig blocksConfig = new() { MinGasPrice = testCase.MinGasPriceForMining }; ITxFilterPipeline txFilterPipeline = new TxFilterPipelineBuilder(LimboLogs.Instance) .WithMinGasPriceFilter(blocksConfig, specProvider) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 29ac38f46e1..83398c4d7b5 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -49,27 +49,36 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi IEip1559Spec specFor1559 = _specProvider.GetSpecFor1559(blockNumber); UInt256 baseFee = BaseFeeCalculator.Calculate(parent, specFor1559); IDictionary pendingTransactions = _transactionPool.GetPendingTransactionsBySender(); + IDictionary pendingBlobTransactionsEquivalences = _transactionPool.GetPendingBlobTransactionsEquivalencesBySender(); IComparer comparer = GetComparer(parent, new BlockPreparationContext(baseFee, blockNumber)) .ThenBy(ByHashTxComparer.Instance); // in order to sort properly and not lose transactions we need to differentiate on their identity which provided comparer might not be doing IEnumerable transactions = GetOrderedTransactions(pendingTransactions, comparer); - IEnumerable blobTransactions = _transactionPool.GetPendingBlobTransactions(); + IEnumerable blobTransactionsEquivalents = GetOrderedTransactions(pendingBlobTransactionsEquivalences, comparer); if (_logger.IsDebug) _logger.Debug($"Collecting pending transactions at block gas limit {gasLimit}."); int selectedTransactions = 0; int i = 0; int blobsCounter = 0; UInt256 blobGasPrice = UInt256.Zero; - List? selectedBlobTxs = null; + using ArrayPoolList selectedBlobTxs = new(Eip4844Constants.MaxBlobsPerBlock); - foreach (Transaction blobTx in blobTransactions) + foreach (Transaction blobTxEquivalent in blobTransactionsEquivalents) { if (blobsCounter == Eip4844Constants.MaxBlobsPerBlock) { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); + if (_logger.IsTrace) _logger.Trace($"Declining {blobTxEquivalent.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); break; } + if (blobTxEquivalent.Hash is null + || !_transactionPool.TryGetPendingBlobTransaction(blobTxEquivalent.Hash, out Transaction blobTx)) + { + continue; + } + + // Transaction blobTx = blobTxEquivalent; + i++; bool success = _txFilterPipeline.Execute(blobTx, parent); @@ -109,7 +118,6 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {blobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); selectedTransactions++; - selectedBlobTxs ??= new List((int)(Eip4844Constants.MaxBlobGasPerBlock / Eip4844Constants.BlobGasPerBlob)); selectedBlobTxs.Add(blobTx); } @@ -127,20 +135,31 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi bool success = _txFilterPipeline.Execute(tx, parent); if (!success) continue; - if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); - - if (selectedBlobTxs?.Count > 0) + if (selectedBlobTxs.Count > 0) { - foreach (Transaction blobTx in new List(selectedBlobTxs)) + using ArrayPoolList txsToRemove = new(selectedBlobTxs.Count); + + foreach (Transaction blobTx in selectedBlobTxs) { if (comparer.Compare(blobTx, tx) > 0) { yield return blobTx; - selectedBlobTxs.Remove(blobTx); + txsToRemove.Add(blobTx); + } + else + { + break; } } + + foreach (Transaction txToRemove in txsToRemove) + { + selectedBlobTxs.Remove(txToRemove); + } } + if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); + selectedTransactions++; yield return tx; } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 8f761689a4a..9f93ff2691a 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -3,7 +3,6 @@ using System; using System.Collections.Generic; -using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Logging; @@ -23,43 +22,6 @@ public BlobTxDistinctSortedPool(int capacity, IComparer comparer, I protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - [MethodImpl(MethodImplOptions.Synchronized)] - public virtual IEnumerable GetBlobTransactions() - { - int pickedBlobs = 0; - List? blobTxsToReadd = null; - - while (pickedBlobs < Eip4844Constants.MaxBlobsPerBlock) - { - Transaction? bestTx = GetFirsts().Min; - - if (bestTx?.Hash is null || bestTx.BlobVersionedHashes is null) - { - break; - } - - if (pickedBlobs + bestTx.BlobVersionedHashes.Length <= Eip4844Constants.MaxBlobsPerBlock) - { - yield return bestTx; - pickedBlobs += bestTx.BlobVersionedHashes.Length; - } - - if (TryRemove(bestTx.Hash)) - { - blobTxsToReadd ??= new(Eip4844Constants.MaxBlobsPerBlock); - blobTxsToReadd.Add(bestTx!); - } - } - - if (blobTxsToReadd is not null) - { - foreach (Transaction blobTx in blobTxsToReadd) - { - TryInsert(blobTx.Hash, blobTx!, out Transaction? removed); - } - } - } - public override void EnsureCapacity() { base.EnsureCapacity(); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index ca72bd1762d..213b04faed2 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Runtime.CompilerServices; using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Crypto; @@ -72,45 +71,6 @@ public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Trans return false; } - [MethodImpl(MethodImplOptions.Synchronized)] - public override IEnumerable GetBlobTransactions() - { - int pickedBlobs = 0; - List? blobTxsToReadd = null; - - while (pickedBlobs < Eip4844Constants.MaxBlobsPerBlock) - { - Transaction? bestTxLight = GetFirsts().Min; - - if (bestTxLight?.Hash is null) - { - break; - } - - if (TryGetValue(bestTxLight.Hash, out Transaction? fullBlobTx) && pickedBlobs + fullBlobTx.BlobVersionedHashes!.Length <= Eip4844Constants.MaxBlobsPerBlock) - { - yield return fullBlobTx!; - pickedBlobs += fullBlobTx.BlobVersionedHashes.Length; - } - - if (TryRemove(bestTxLight.Hash)) - { - blobTxsToReadd ??= new(); - blobTxsToReadd.Add(fullBlobTx!); - } - } - - - if (blobTxsToReadd is not null) - { - foreach (Transaction fullBlobTx in blobTxsToReadd) - { - TryInsert(fullBlobTx.Hash, fullBlobTx!, out Transaction? removed); - } - } - - } - protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 96979608ea7..45ee33fe1da 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -21,10 +21,10 @@ public interface ITxPool IDictionary GetPendingTransactionsBySender(); /// - /// Lazy enumerate blob txs starting from the best one + /// Blob txs light equivalences grouped by sender address, sorted by nonce and later tx pool sorting /// /// - IEnumerable GetPendingBlobTransactions(); + IDictionary GetPendingBlobTransactionsEquivalencesBySender(); /// /// from a specific sender, sorted by nonce and later tx pool sorting @@ -37,6 +37,7 @@ public interface ITxPool bool RemoveTransaction(Keccak? hash); bool IsKnown(Keccak hash); bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction); + bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction); UInt256 GetLatestPendingNonce(Address address); event EventHandler NewDiscovered; event EventHandler NewPending; diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 30f9c4e5f51..19547e8282e 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -19,11 +19,13 @@ private NullTxPool() { } public Transaction[] GetPendingTransactions() => Array.Empty(); - public Transaction[] GetOwnPendingTransactions() => Array.Empty(); - public Transaction[] GetPendingTransactionsBySender(Address address) => Array.Empty(); - public IDictionary GetPendingTransactionsBySender() => new Dictionary(); + public IDictionary GetPendingTransactionsBySender() + => new Dictionary(); + + public IDictionary GetPendingBlobTransactionsEquivalencesBySender() + => new Dictionary(); public IEnumerable GetPendingBlobTransactions() => Array.Empty(); @@ -43,7 +45,12 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) return false; } - public UInt256 ReserveOwnTransactionNonce(Address address) => UInt256.Zero; + public bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction) + { + blobTransaction = null; + return false; + } + public UInt256 GetLatestPendingNonce(Address address) => 0; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index b401bc0e9fb..f00a7be4982 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -156,14 +156,15 @@ public TxPool(IEthereumEcdsa ecdsa, public IDictionary GetPendingTransactionsBySender() => _transactions.GetBucketSnapshot(); + public IDictionary GetPendingBlobTransactionsEquivalencesBySender() => + _blobTransactions.GetBucketSnapshot(); + public Transaction[] GetPendingTransactionsBySender(Address address) => _transactions.GetBucketSnapshot(address); // only for testing reasons internal Transaction[] GetOwnPendingTransactions() => _broadcaster.GetSnapshot(); - public IEnumerable GetPendingBlobTransactions() => _blobTransactions.GetBlobTransactions(); - public int GetPendingBlobTransactionsCount() => _blobTransactions.Count; private void OnHeadChange(object? sender, BlockReplacementEventArgs e) @@ -596,6 +597,14 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) } } + public bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction) + { + lock (_locker) + { + return _blobTransactions.TryGetValue(hash, out blobTransaction); + } + } + // only for tests - to test sorting internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? transaction) => _blobTransactions.TryGetBlobTxSortingEquivalent(hash, out transaction); From e43ad49dfd68f0291a7af5e2bf498a80afc496b9 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 25 Aug 2023 13:43:13 +0200 Subject: [PATCH 088/148] fix whitespace --- .../Nethermind.Blockchain.Test/TransactionSelectorTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index 72bd8edb5bb..7fcf6ca8844 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -247,7 +247,7 @@ void SetAccountStates(IEnumerable
missingAddresses) g => g.OrderBy(t => t, comparer).ToArray()); transactionPool.GetPendingTransactionsBySender().Returns(transactions); transactionPool.GetPendingBlobTransactionsEquivalencesBySender().Returns(blobTransactions); - foreach (KeyValuePair keyValuePair in blobTransactions) + foreach (KeyValuePair keyValuePair in blobTransactions) { foreach (Transaction blobTx in keyValuePair.Value) { From b3a5ffe5310051052cccd75b489c89451098bf46 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 25 Aug 2023 14:48:05 +0200 Subject: [PATCH 089/148] simplify decoding in BlobTxStorage --- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 11 ++--------- src/Nethermind/Nethermind.TxPool/TxPool.cs | 1 - 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 6312c263333..dd80615ba00 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -32,15 +32,8 @@ private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) { if (txBytes is not null) { - try - { - transaction = Rlp.Decode(new Rlp(txBytes), RlpBehaviors.InMempoolForm); - return true; - } - catch (Exception) - { - // ignored - } + transaction = Rlp.Decode(txBytes, RlpBehaviors.InMempoolForm); + return true; } transaction = default; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index f00a7be4982..db9c189b801 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -76,7 +76,6 @@ public class TxPool : ITxPool, IDisposable /// /// /// - /// Tx storage used to reject known transactions. public TxPool(IEthereumEcdsa ecdsa, ITxStorage blobTxStorage, IChainHeadInfoProvider chainHeadInfoProvider, From 22d8de732565774dc3d00ba603c635de7da59a6b Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 25 Aug 2023 15:20:57 +0200 Subject: [PATCH 090/148] cosmetics --- .../Nethermind.Consensus/Producers/TxPoolTxSource.cs | 2 -- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 3 ++- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 3 ++- src/Nethermind/Nethermind.TxPool/TxPool.cs | 3 ++- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 83398c4d7b5..fca225c6b38 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -77,8 +77,6 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi continue; } - // Transaction blobTx = blobTxEquivalent; - i++; bool success = _txFilterPipeline.Execute(blobTx, parent); diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 45ee33fe1da..705b5653359 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -37,7 +38,7 @@ public interface ITxPool bool RemoveTransaction(Keccak? hash); bool IsKnown(Keccak hash); bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction); - bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction); + bool TryGetPendingBlobTransaction(Keccak hash, [NotNullWhen(true)] out Transaction? blobTransaction); UInt256 GetLatestPendingNonce(Address address); event EventHandler NewDiscovered; event EventHandler NewPending; diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 19547e8282e..075720d9626 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -45,7 +46,7 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) return false; } - public bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction) + public bool TryGetPendingBlobTransaction(Keccak hash, [NotNullWhen(true)] out Transaction? blobTransaction) { blobTransaction = null; return false; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index db9c189b801..20857e322b2 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; using System.Threading; @@ -596,7 +597,7 @@ public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) } } - public bool TryGetPendingBlobTransaction(Keccak hash, out Transaction? blobTransaction) + public bool TryGetPendingBlobTransaction(Keccak hash, [NotNullWhen(true)] out Transaction? blobTransaction) { lock (_locker) { From 2488460752d1c925e34b24c9f1051dbd17d6284c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Sat, 26 Aug 2023 14:14:29 +0200 Subject: [PATCH 091/148] simplify tests --- .../TxBroadcasterTests.cs | 144 +++++++++--------- 1 file changed, 72 insertions(+), 72 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs index b5a112d98cf..4c1daef9c3d 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxBroadcasterTests.cs @@ -202,6 +202,7 @@ public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast( _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); _headInfo.CurrentBaseFee.Returns(0.GWei()); + // add 256 transactions, 10% of them is large int addedTxsCount = TestItem.PrivateKeys.Length; Transaction[] transactions = new Transaction[addedTxsCount]; @@ -220,48 +221,24 @@ public void should_skip_large_txs_when_picking_best_persistent_txs_to_broadcast( _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); - (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); - + // count numbers of expected hashes and full transactions int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); int expectedCountOfHashes = expectedCountTotal / 10 + 1; int expectedCountOfFullTxs = expectedCountTotal - expectedCountOfHashes; - if (expectedCountOfFullTxs > 0) - { - pickedTxs.Count.Should().Be(expectedCountOfFullTxs); - } - else - { - pickedTxs.Should().BeNull(); - } - if (expectedCountOfHashes > 0) - { - pickedHashes.Count.Should().Be(expectedCountOfHashes); - } - else - { - pickedHashes.Should().BeNull(); - } + // prepare list of expected full transactions and hashes + (IList expectedFullTxs, IList expectedHashes) = GetTxsAndHashesExpectedToBroadcast(transactions, expectedCountTotal); - List expectedTxs = new(); - List expectedHashes = new(); + // get hashes and full transactions to broadcast + (IList pickedFullTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); - for (int i = 0; i < expectedCountTotal; i++) - { - Transaction tx = transactions[i]; + // check if numbers of full transactions and hashes are correct + CheckCorrectness(pickedFullTxs, expectedCountOfFullTxs); + CheckCorrectness(pickedHashes, expectedCountOfHashes); - if (tx.CanBeBroadcast()) - { - expectedTxs.Add(tx); - } - else - { - expectedHashes.Add(tx); - } - } - - expectedTxs.Should().BeEquivalentTo(pickedTxs); - expectedHashes.Should().BeEquivalentTo(pickedHashes); + // check if full transactions and hashes returned by broadcaster are as expected + expectedFullTxs.Should().BeEquivalentTo(pickedFullTxs); + expectedHashes.Should().BeEquivalentTo(pickedHashes.Select(t => t.Hash).ToArray()); } [Test] @@ -270,6 +247,8 @@ public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast([ _txPoolConfig = new TxPoolConfig() { PeerNotificationThreshold = threshold }; _broadcaster = new TxBroadcaster(_comparer, TimerFactory.Default, _txPoolConfig, _headInfo, _logManager); _headInfo.CurrentBaseFee.Returns(0.GWei()); + + // add 256 transactions, 10% of them is blob type int addedTxsCount = TestItem.PrivateKeys.Length; Transaction[] transactions = new Transaction[addedTxsCount]; @@ -285,46 +264,27 @@ public void should_skip_blob_txs_when_picking_best_persistent_txs_to_broadcast([ _broadcaster.Broadcast(transactions[i], true); } + _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); - (IList pickedTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); + // count numbers of expected hashes and full transactions int expectedCountTotal = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount); int expectedCountOfBlobHashes = expectedCountTotal / 10 + 1; int expectedCountOfNonBlobTxs = expectedCountTotal - expectedCountOfBlobHashes; - if (expectedCountOfNonBlobTxs > 0) - { - pickedTxs.Count.Should().Be(expectedCountOfNonBlobTxs); - } - else - { - pickedTxs.Should().BeNull(); - } - if (expectedCountOfBlobHashes > 0) - { - pickedHashes.Count.Should().Be(expectedCountOfBlobHashes); - } - else - { - pickedHashes.Should().BeNull(); - } - List expectedTxs = new(); - List expectedHashes = new(); - for (int i = 0; i < expectedCountTotal; i++) - { - Transaction tx = transactions[i]; + // prepare list of expected full transactions and hashes + (IList expectedFullTxs, IList expectedHashes) = GetTxsAndHashesExpectedToBroadcast(transactions, expectedCountTotal); - if (!tx.SupportsBlobs) - { - expectedTxs.Add(tx); - } - else - { - expectedHashes.Add(new LightTransaction(tx)); - } - } - expectedTxs.Should().BeEquivalentTo(pickedTxs); - expectedHashes.Should().BeEquivalentTo(pickedHashes); + // get hashes and full transactions to broadcast + (IList pickedFullTxs, IList pickedHashes) = _broadcaster.GetPersistentTxsToSend(); + + // check if numbers of full transactions and hashes are correct + CheckCorrectness(pickedFullTxs, expectedCountOfNonBlobTxs); + CheckCorrectness(pickedHashes, expectedCountOfBlobHashes); + + // check if full transactions and hashes returned by broadcaster are as expected + expectedFullTxs.Should().BeEquivalentTo(pickedFullTxs); + expectedHashes.Should().BeEquivalentTo(pickedHashes.Select(t => t.Hash).ToArray()); } [Test] @@ -425,6 +385,7 @@ public void should_not_pick_blob_txs_with_MaxFeePerBlobGas_lower_than_CurrentPri const int currentPricePerBlobGasInGwei = 250; _headInfo.CurrentPricePerBlobGas.Returns(currentPricePerBlobGasInGwei.GWei()); + // add 256 transactions with MaxFeePerBlobGas 0-255 int addedTxsCount = TestItem.PrivateKeys.Length; Transaction[] transactions = new Transaction[addedTxsCount]; @@ -441,18 +402,23 @@ public void should_not_pick_blob_txs_with_MaxFeePerBlobGas_lower_than_CurrentPri _broadcaster.GetSnapshot().Length.Should().Be(addedTxsCount); - IList pickedTxs = _broadcaster.GetPersistentTxsToSend().HashesToSend; - + // count number of expected hashes to broadcast int expectedCount = Math.Min(addedTxsCount * threshold / 100 + 1, addedTxsCount - currentPricePerBlobGasInGwei); - pickedTxs.Count.Should().Be(expectedCount); + // prepare list of expected hashes to broadcast List expectedTxs = new(); - for (int i = 1; i <= expectedCount; i++) { expectedTxs.Add(transactions[addedTxsCount - i]); } + // get actual hashes to broadcast + IList pickedHashes = _broadcaster.GetPersistentTxsToSend().HashesToSend; + + // check if number of hashes to broadcast is correct + pickedHashes.Count.Should().Be(expectedCount); + + // check if number of hashes to broadcast (with MaxFeePerBlobGas >= current) is correct expectedTxs.Count(t => t.MaxFeePerBlobGas >= (UInt256)currentPricePerBlobGasInGwei).Should().Be(expectedCount); } @@ -687,4 +653,38 @@ public void should_rebroadcast_all_persistent_transactions_if_PeerNotificationTh pickedTxs[i].Nonce.Should().Be((UInt256)i); } } + + private (IList expectedTxs, IList expectedHashes) GetTxsAndHashesExpectedToBroadcast(Transaction[] transactions, int expectedCountTotal) + { + List expectedTxs = new(); + List expectedHashes = new(); + + for (int i = 0; i < expectedCountTotal; i++) + { + Transaction tx = transactions[i]; + + if (tx.CanBeBroadcast()) + { + expectedTxs.Add(tx); + } + else + { + expectedHashes.Add(tx.Hash); + } + } + + return (expectedTxs, expectedHashes); + } + + private void CheckCorrectness(IList pickedTxs, int expectedCount) + { + if (expectedCount > 0) + { + pickedTxs.Count.Should().Be(expectedCount); + } + else + { + pickedTxs.Should().BeNull(); + } + } } From 5b78cf4534e8cdd64d24a5974191c38b6b88be30 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Sat, 26 Aug 2023 14:51:48 +0200 Subject: [PATCH 092/148] fix tx priority? --- src/Nethermind/Nethermind.TxPool/LightTransaction.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index c37375cae24..76593e45626 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -24,6 +24,8 @@ public LightTransaction(Transaction fullTx) MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; GasBottleneck = fullTx.GasBottleneck; + Timestamp = fullTx.Timestamp; + PoolIndex = fullTx.PoolIndex; _size = fullTx.GetLength(); } } From d0bc31938414d0c16c8cec87a9852f675dc7881e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Sat, 26 Aug 2023 15:35:24 +0200 Subject: [PATCH 093/148] improve readability in NonceGapFilter --- src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs index e0566385bc4..0acd2ec69d8 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/GapNonceFilter.cs @@ -28,7 +28,8 @@ public GapNonceFilter(TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs, IL public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; - if ((isLocal || !_txs.IsFull()) && !tx.SupportsBlobs) + bool nonceGapsAllowed = isLocal || !_txs.IsFull(); + if (nonceGapsAllowed && !tx.SupportsBlobs) { return AcceptTxResult.Accepted; } From 6ffeb44ea5e180ecd42af39e9651d405edc55b5d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Sat, 26 Aug 2023 17:52:29 +0200 Subject: [PATCH 094/148] improve readability of TxPoolTxSource --- .../Producers/TxPoolTxSource.cs | 122 +++++++++++------- 1 file changed, 74 insertions(+), 48 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index fca225c6b38..81c7f1243e1 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Runtime.CompilerServices; using Nethermind.Consensus.Comparers; @@ -54,7 +55,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi .ThenBy(ByHashTxComparer.Instance); // in order to sort properly and not lose transactions we need to differentiate on their identity which provided comparer might not be doing IEnumerable transactions = GetOrderedTransactions(pendingTransactions, comparer); - IEnumerable blobTransactionsEquivalents = GetOrderedTransactions(pendingBlobTransactionsEquivalences, comparer); + IEnumerable blobTransactions = GetOrderedTransactions(pendingBlobTransactionsEquivalences, comparer); if (_logger.IsDebug) _logger.Debug($"Collecting pending transactions at block gas limit {gasLimit}."); int selectedTransactions = 0; @@ -63,60 +64,48 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi UInt256 blobGasPrice = UInt256.Zero; using ArrayPoolList selectedBlobTxs = new(Eip4844Constants.MaxBlobsPerBlock); - foreach (Transaction blobTxEquivalent in blobTransactionsEquivalents) + foreach (Transaction blobTx in blobTransactions) { if (blobsCounter == Eip4844Constants.MaxBlobsPerBlock) { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTxEquivalent.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, no more blob space. Block already have {blobsCounter} which is max value allowed."); break; } - if (blobTxEquivalent.Hash is null - || !_transactionPool.TryGetPendingBlobTransaction(blobTxEquivalent.Hash, out Transaction blobTx)) + if (!TryGetFullBlobTx(blobTx, out Transaction fullBlobTx)) { + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to get full version of this blob tx from TxPool."); continue; } i++; - bool success = _txFilterPipeline.Execute(blobTx, parent); + bool success = _txFilterPipeline.Execute(fullBlobTx, parent); if (!success) continue; - if (blobGasPrice.IsZero) + if (blobGasPrice.IsZero && !TryUpdateBlobGasPrice(fullBlobTx, parent, out blobGasPrice)) { - ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, _specProvider.GetSpec(parent)); - if (excessDataGas is null) - { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); - continue; - } - if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(excessDataGas.Value, out blobGasPrice)) - { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to calculate data gas price."); - continue; - } + continue; } - int txAmountOfBlobs = blobTx.BlobVersionedHashes?.Length ?? 0; - - if (blobGasPrice > blobTx.MaxFeePerBlobGas) + if (blobGasPrice > fullBlobTx.MaxFeePerBlobGas) { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, data gas fee is too low."); + if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, data gas fee is too low."); continue; } - if (BlobGasCalculator.CalculateBlobGas(blobsCounter + txAmountOfBlobs) > - Eip4844Constants.MaxBlobGasPerBlock) + int txAmountOfBlobs = fullBlobTx.BlobVersionedHashes?.Length ?? 0; + if (blobsCounter + txAmountOfBlobs > Eip4844Constants.MaxBlobsPerBlock) { - if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, not enough blob space."); + if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, not enough blob space."); continue; } blobsCounter += txAmountOfBlobs; - if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {blobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); + if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {fullBlobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); selectedTransactions++; - selectedBlobTxs.Add(blobTx); + selectedBlobTxs.Add(fullBlobTx); } foreach (Transaction tx in transactions) @@ -133,27 +122,9 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi bool success = _txFilterPipeline.Execute(tx, parent); if (!success) continue; - if (selectedBlobTxs.Count > 0) + foreach (Transaction blobTx in PickBlobTxsBetterThanCurrentTx(selectedBlobTxs, tx, comparer)) { - using ArrayPoolList txsToRemove = new(selectedBlobTxs.Count); - - foreach (Transaction blobTx in selectedBlobTxs) - { - if (comparer.Compare(blobTx, tx) > 0) - { - yield return blobTx; - txsToRemove.Add(blobTx); - } - else - { - break; - } - } - - foreach (Transaction txToRemove in txsToRemove) - { - selectedBlobTxs.Remove(txToRemove); - } + yield return blobTx; } if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); @@ -162,7 +133,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi yield return tx; } - if (selectedBlobTxs?.Count > 0) + if (selectedBlobTxs.Count > 0) { foreach (Transaction blobTx in selectedBlobTxs) { @@ -173,6 +144,61 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi if (_logger.IsDebug) _logger.Debug($"Potentially selected {selectedTransactions} out of {i} pending transactions checked."); } + private bool TryGetFullBlobTx(Transaction blobTx, [NotNullWhen(true)] out Transaction? fullBlobTx) + { + if (blobTx.NetworkWrapper is not null) + { + fullBlobTx = blobTx; + return true; + } + fullBlobTx = null; + return blobTx.Hash is not null && _transactionPool.TryGetPendingBlobTransaction(blobTx.Hash, out fullBlobTx); + } + + private bool TryUpdateBlobGasPrice(Transaction fullBlobTx, BlockHeader parent, out UInt256 blobGasPrice) + { + ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, _specProvider.GetSpec(parent)); + if (excessDataGas is null) + { + if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); + blobGasPrice = UInt256.Zero; + return false; + } + if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(excessDataGas.Value, out blobGasPrice)) + { + if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, failed to calculate data gas price."); + blobGasPrice = UInt256.Zero; + return false; + } + return true; + } + + private IEnumerable PickBlobTxsBetterThanCurrentTx(ArrayPoolList selectedBlobTxs, Transaction tx, IComparer comparer) + { + if (selectedBlobTxs.Count > 0) + { + using ArrayPoolList txsToRemove = new(selectedBlobTxs.Count); + + foreach (Transaction blobTx in selectedBlobTxs) + { + if (comparer.Compare(blobTx, tx) > 0) + { + yield return blobTx; + txsToRemove.Add(blobTx); + } + else + { + break; + } + } + + foreach (Transaction txToRemove in txsToRemove) + { + selectedBlobTxs.Remove(txToRemove); + } + } + } + protected virtual IEnumerable GetOrderedTransactions(IDictionary pendingTransactions, IComparer comparer) => Order(pendingTransactions, comparer); From 416ab31bc0e1c69c95dfc15c0e2a93cf80465942 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 11 Sep 2023 12:31:17 +0200 Subject: [PATCH 095/148] save sender to db --- src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index dd80615ba00..52ea91216fa 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -4,8 +4,10 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using System.Linq; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Core.Extensions; using Nethermind.Db; using Nethermind.Serialization.Rlp; @@ -32,7 +34,10 @@ private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) { if (txBytes is not null) { - transaction = Rlp.Decode(txBytes, RlpBehaviors.InMempoolForm); + RlpStream rlpStream = new(txBytes); + Address sender = new(rlpStream.Read(20).ToArray()); + transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); + transaction.SenderAddress = sender; return true; } @@ -58,7 +63,11 @@ public void Add(Transaction transaction) throw new ArgumentNullException(nameof(transaction)); } - _database.Set(transaction.Hash, Rlp.Encode(transaction, RlpBehaviors.InMempoolForm).Bytes); + _database.Set(transaction.Hash, + Bytes.Concat( + transaction.SenderAddress!.Bytes, + Rlp.Encode(transaction, RlpBehaviors.InMempoolForm).Bytes + ).ToArray()); } public void Delete(ValueKeccak hash) From c236326e066eb60a2e5a558db02988dce31f84f1 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 11 Sep 2023 12:37:05 +0200 Subject: [PATCH 096/148] fix recreating light collection --- .../PersistentBlobTxDistinctSortedPool.cs | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 213b04faed2..6adf53d5db5 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Caching; @@ -30,13 +31,26 @@ public PersistentBlobTxDistinctSortedPool(ITxStorage blobTxStorage, ITxPoolConfi private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) { if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); + int numberOfTxsInDb = 0; + int numberOfBlobsInDb = 0; + Stopwatch stopwatch = new(); + stopwatch.Start(); foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) { - if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx))) + if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out _)) { _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); + numberOfTxsInDb++; + numberOfBlobsInDb += fullBlobTx.BlobVersionedHashes?.Length ?? 0; } } + + if (_logger.IsInfo && numberOfTxsInDb != 0) + { + long loadingTime = stopwatch.ElapsedMilliseconds; + _logger.Info($"Loaded {numberOfTxsInDb} blob txs from persistent db, containing {numberOfBlobsInDb} blobs, in {loadingTime}ms"); + } + stopwatch.Stop(); } public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Transaction? removed) From 1b5c1ccb8e776ab3c7af34145dc1408bb181dbfe Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 11:45:50 +0200 Subject: [PATCH 097/148] adjust EthStats to show sum of blob and non-blob txs --- .../Nethermind.EthStats/Integrations/EthStatsIntegration.cs | 2 +- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.EthStats/Integrations/EthStatsIntegration.cs b/src/Nethermind/Nethermind.EthStats/Integrations/EthStatsIntegration.cs index 3c02c95b417..2d88849fb38 100644 --- a/src/Nethermind/Nethermind.EthStats/Integrations/EthStatsIntegration.cs +++ b/src/Nethermind/Nethermind.EthStats/Integrations/EthStatsIntegration.cs @@ -137,7 +137,7 @@ private void TimerOnElapsed(object? sender, ElapsedEventArgs e) { if (_logger.IsDebug) _logger.Debug("ETH Stats sending 'stats' message..."); SendStatsAsync(); - SendPendingAsync(_txPool.GetPendingTransactionsCount()); + SendPendingAsync(_txPool.GetPendingTransactionsCount() + _txPool.GetPendingBlobTransactionsCount()); } } diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index 705b5653359..e03cdb85a9f 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -13,6 +13,7 @@ namespace Nethermind.TxPool public interface ITxPool { int GetPendingTransactionsCount(); + int GetPendingBlobTransactionsCount(); Transaction[] GetPendingTransactions(); /// From 051c89b44d8012debe0e0238b556aeb753c5d5be Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 11:51:13 +0200 Subject: [PATCH 098/148] fix --- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 075720d9626..52ef615a984 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -17,7 +17,7 @@ private NullTxPool() { } public static NullTxPool Instance { get; } = new(); public int GetPendingTransactionsCount() => 0; - + public int GetPendingBlobTransactionsCount() => 0; public Transaction[] GetPendingTransactions() => Array.Empty(); public Transaction[] GetPendingTransactionsBySender(Address address) => Array.Empty(); From d16bf52c1734d1772cf5acfa34cb0a604ae5362c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 17:40:31 +0200 Subject: [PATCH 099/148] make blob support configurable --- .../FullPruning/FullPruningDiskTest.cs | 2 +- .../StandardDbInitializerTests.cs | 6 ++-- .../Nethermind.Db/StandardDbInitializer.cs | 20 +++++------ .../Nethermind.Init/Steps/InitDatabase.cs | 8 +++-- .../TxPoolTests.Blobs.cs | 19 +++++++++++ .../Nethermind.TxPool/AcceptTxResult.cs | 5 +++ .../Collections/BlobTxDistinctSortedPool.cs | 4 +-- .../Filters/NotSupportedTxFilter.cs | 33 +++++++++++++++++++ .../Nethermind.TxPool/ITxPoolConfig.cs | 3 ++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 5 +-- .../Nethermind.TxPool/TxPoolConfig.cs | 1 + 11 files changed, 84 insertions(+), 22 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs diff --git a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs index 8e3e3f28453..25aab06c075 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/FullPruning/FullPruningDiskTest.cs @@ -61,7 +61,7 @@ protected override async Task CreateDbProvider() { IDbProvider dbProvider = new DbProvider(DbModeHint.Persisted); RocksDbFactory rocksDbFactory = new(new DbConfig(), LogManager, TempDirectory.Path); - StandardDbInitializer standardDbInitializer = new(dbProvider, rocksDbFactory, new MemDbFactory(), new FileSystem(), true); + StandardDbInitializer standardDbInitializer = new(dbProvider, rocksDbFactory, new MemDbFactory(), new FileSystem()); await standardDbInitializer.InitStandardDbsAsync(true); return dbProvider; } diff --git a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs index 91a4845eb38..65fb1da0d0f 100644 --- a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs @@ -62,15 +62,15 @@ public async Task InitializerTests_ReadonlyDbProvider(bool useReceipts) [Test] public async Task InitializerTests_WithPruning() { - IDbProvider dbProvider = await InitializeStandardDb(false, DbModeHint.Mem, "pruning", true); + IDbProvider dbProvider = await InitializeStandardDb(false, DbModeHint.Mem, "pruning"); dbProvider.StateDb.Should().BeOfType(); } - private async Task InitializeStandardDb(bool useReceipts, DbModeHint dbModeHint, string path, bool pruning = false) + private async Task InitializeStandardDb(bool useReceipts, DbModeHint dbModeHint, string path) { using IDbProvider dbProvider = new DbProvider(dbModeHint); RocksDbFactory rocksDbFactory = new(new DbConfig(), LimboLogs.Instance, Path.Combine(_folderWithDbs, path)); - StandardDbInitializer initializer = new(dbProvider, rocksDbFactory, new MemDbFactory(), Substitute.For(), pruning); + StandardDbInitializer initializer = new(dbProvider, rocksDbFactory, new MemDbFactory(), Substitute.For()); await initializer.InitStandardDbsAsync(useReceipts); return dbProvider; } diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index b6ff4e65055..60298e95abc 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -12,33 +12,30 @@ namespace Nethermind.Db public class StandardDbInitializer : RocksDbInitializer { private readonly IFileSystem _fileSystem; - private readonly bool _fullPruning; public StandardDbInitializer( IDbProvider? dbProvider, IRocksDbFactory? rocksDbFactory, IMemDbFactory? memDbFactory, - IFileSystem? fileSystem = null, - bool fullPruning = false) + IFileSystem? fileSystem = null) : base(dbProvider, rocksDbFactory, memDbFactory) { _fileSystem = fileSystem ?? new FileSystem(); - _fullPruning = fullPruning; } - public void InitStandardDbs(bool useReceiptsDb) + public void InitStandardDbs(bool useReceiptsDb, bool useBlobDb = true) { - RegisterAll(useReceiptsDb); + RegisterAll(useReceiptsDb, useBlobDb); InitAll(); } - public async Task InitStandardDbsAsync(bool useReceiptsDb) + public async Task InitStandardDbsAsync(bool useReceiptsDb, bool useBlobDb = true) { - RegisterAll(useReceiptsDb); + RegisterAll(useReceiptsDb, useBlobDb); await InitAllAsync(); } - private void RegisterAll(bool useReceiptsDb) + private void RegisterAll(bool useReceiptsDb, bool useBlobDb) { RegisterDb(BuildRocksDbSettings(DbNames.Blocks, () => Metrics.BlocksDbReads++, () => Metrics.BlocksDbWrites++)); RegisterDb(BuildRocksDbSettings(DbNames.Headers, () => Metrics.HeaderDbReads++, () => Metrics.HeaderDbWrites++)); @@ -65,7 +62,10 @@ private void RegisterAll(bool useReceiptsDb) RegisterCustomDb(DbNames.Receipts, () => new ReadOnlyColumnsDb(new MemColumnsDb(), false)); } RegisterDb(BuildRocksDbSettings(DbNames.Metadata, () => Metrics.MetadataDbReads++, () => Metrics.MetadataDbWrites++)); - RegisterDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); + if (useBlobDb) + { + RegisterDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); + } } private RocksDbSettings BuildRocksDbSettings(string dbName, Action updateReadsMetrics, Action updateWriteMetrics, bool deleteOnStart = false) diff --git a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs index 3b1bda2069b..179fbcd60ee 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs @@ -14,6 +14,7 @@ using Nethermind.Db.Rpc; using Nethermind.JsonRpc.Client; using Nethermind.Logging; +using Nethermind.TxPool; namespace Nethermind.Init.Steps { @@ -35,7 +36,7 @@ public async Task Execute(CancellationToken _) IDbConfig dbConfig = _api.Config(); ISyncConfig syncConfig = _api.Config(); IInitConfig initConfig = _api.Config(); - IPruningConfig pruningConfig = _api.Config(); + ITxPoolConfig txPoolConfig = _api.Config(); foreach (PropertyInfo propertyInfo in typeof(IDbConfig).GetProperties()) { @@ -45,9 +46,10 @@ public async Task Execute(CancellationToken _) try { bool useReceiptsDb = initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync; + bool useBlobDb = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true }; InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); - StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem, pruningConfig.Mode.IsFull()); - await dbInitializer.InitStandardDbsAsync(useReceiptsDb); + StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem); + await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobDb); } catch (TypeInitializationException ex) { diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 6c0af30f2cb..be17af9b2ec 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -17,6 +17,25 @@ namespace Nethermind.TxPool.Test [TestFixture] public partial class TxPoolTests { + [Test] + public void should_reject_blob_tx_if_blobs_not_supported([Values(true, false)] bool isBlobSupportEnabled) + { + TxPoolConfig txPoolConfig = new() { BlobSupportEnabled = isBlobSupportEnabled }; + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + + Transaction tx = Build.A.Transaction + .WithNonce(UInt256.Zero) + .WithShardBlobTxTypeAndFields() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + + _txPool.SubmitTx(tx, TxHandlingOptions.PersistentBroadcast).Should().Be(isBlobSupportEnabled + ? AcceptTxResult.Accepted + : AcceptTxResult.NotSupportedTxType); + } + [Test] public void blob_pool_size_should_be_correct([Values(true, false)] bool persistentStorageEnabled) { diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index 941a73f4b8a..fa3bca1d683 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -80,6 +80,11 @@ namespace Nethermind.TxPool /// public static readonly AcceptTxResult PendingTxsOfOtherType = new(13, nameof(PendingTxsOfOtherType)); + /// + /// Ignores transactions if tx type is not supported + /// + public static readonly AcceptTxResult NotSupportedTxType = new(14, nameof(NotSupportedTxType)); + private int Id { get; } private string Code { get; } private string? Message { get; } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 9f93ff2691a..f1e3b49b719 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -24,10 +24,8 @@ protected override IComparer GetReplacementComparer(IComparer _poolCapacity && _logger.IsWarn) - _logger.Warn($"Blob TxPool exceeds the config size {Count}/{_poolCapacity}"); + _logger.Warn($"Blob pool exceeds the config size {Count}/{_poolCapacity}"); } /// diff --git a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs new file mode 100644 index 00000000000..d7191ac4338 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs @@ -0,0 +1,33 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.Logging; + +namespace Nethermind.TxPool.Filters; + +/// +/// Filters out transactions types that are not supported +/// +internal sealed class NotSupportedTxFilter : IIncomingTxFilter +{ + private readonly ITxPoolConfig _txPoolConfig; + private readonly ILogger _logger; + + public NotSupportedTxFilter(ITxPoolConfig txPoolConfig, ILogger logger) + { + _txPoolConfig = txPoolConfig; + _logger = logger; + } + + public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) + { + if (!_txPoolConfig.BlobSupportEnabled && tx.SupportsBlobs) + { + if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, blob transactions are not supported."); + return AcceptTxResult.NotSupportedTxType; + } + + return AcceptTxResult.Accepted; + } +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 851923f2d5f..a8c64d521c1 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -13,6 +13,9 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "Max number of transactions held in mempool (more transactions in mempool mean more memory used")] int Size { get; set; } + [ConfigItem(DefaultValue = "true", Description = "If true, blob transactions support will be enabled")] + bool BlobSupportEnabled { get; set; } + [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] bool PersistentBlobStorageEnabled { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 20857e322b2..905684dfe8c 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -99,15 +99,16 @@ public TxPool(IEthereumEcdsa ecdsa, AddNodeInfoEntryForTxPool(); _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); - _blobTransactions = txPoolConfig.PersistentBlobStorageEnabled + _blobTransactions = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true } ? new PersistentBlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager) - : new BlobTxDistinctSortedPool(_txPoolConfig.InMemoryBlobPoolSize, comparer, logManager); + : new BlobTxDistinctSortedPool(txPoolConfig.BlobSupportEnabled ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); _headInfo.HeadChanged += OnHeadChange; _preHashFilters = new IIncomingTxFilter[] { + new NotSupportedTxFilter(txPoolConfig, _logger), new GasLimitTxFilter(_headInfo, txPoolConfig, _logger), new FeeTooLowFilter(_headInfo, _transactions, _blobTransactions, thereIsPriorityContract, _logger), new MalformedTxFilter(_specProvider, validator, _logger) diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 6acca9f2664..2ca2fb5ddb0 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,6 +7,7 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; + public bool BlobSupportEnabled { get; set; } = true; public bool PersistentBlobStorageEnabled { get; set; } = false; public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 12,5GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. From 6f211b6a577c81fc715ee7686b98c9ea0fbff03d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 17:45:24 +0200 Subject: [PATCH 100/148] optimize snapshots --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 905684dfe8c..bdea9c3c930 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -62,6 +62,7 @@ public class TxPool : ITxPool, IDisposable private readonly ITimer? _timer; private Transaction[]? _transactionSnapshot; + private Transaction[]? _blobTransactionSnapshot; /// /// This class stores all known pending transactions that can be used for block production @@ -174,6 +175,7 @@ private void OnHeadChange(object? sender, BlockReplacementEventArgs e) { // Clear snapshot _transactionSnapshot = null; + _blobTransactionSnapshot = null; _hashCache.ClearCurrentBlockCache(); _headBlocksChannel.Writer.TryWrite(e); } @@ -294,7 +296,8 @@ public void AddPeer(ITxPoolPeer peer) if (_broadcaster.AddPeer(peer)) { // worth to refactor and prepare tx snapshot in more efficient way - _broadcaster.BroadcastOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot().Concat(_blobTransactions.GetSnapshot()).ToArray()); + _broadcaster.BroadcastOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot()); + _broadcaster.BroadcastOnce(peer, _blobTransactionSnapshot ??= _blobTransactions.GetSnapshot()); if (_logger.IsTrace) _logger.Trace($"Added a peer to TX pool: {peer}"); } @@ -339,8 +342,11 @@ public AcceptTxResult SubmitTx(Transaction tx, TxHandlingOptions handlingOptions accepted = AddCore(tx, state, startBroadcast); if (accepted) { - // Clear snapshot - _transactionSnapshot = null; + // Clear proper snapshot + if (tx.SupportsBlobs) + _blobTransactionSnapshot = null; + else + _transactionSnapshot = null; } } From 96352c88d3e4457b858d9d6c4fa03c8c9b625a34 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 19:27:36 +0200 Subject: [PATCH 101/148] fix blob support disabled --- .../Nethermind.Api/IApiWithStores.cs | 2 ++ .../Nethermind.Api/NethermindApi.cs | 1 + .../InitializeBlockchainAuRa.cs | 2 +- .../Nethermind.Init/Steps/InitDatabase.cs | 3 ++ .../Steps/InitializeBlockchain.cs | 2 +- .../Steps/InitializeNetwork.cs | 3 +- .../Nethermind.TxPool/NullBlobTxStorage.cs | 30 +++++++++++++++++++ 7 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs diff --git a/src/Nethermind/Nethermind.Api/IApiWithStores.cs b/src/Nethermind/Nethermind.Api/IApiWithStores.cs index 3c0f71b8c1d..12fe585bc59 100644 --- a/src/Nethermind/Nethermind.Api/IApiWithStores.cs +++ b/src/Nethermind/Nethermind.Api/IApiWithStores.cs @@ -8,12 +8,14 @@ using Nethermind.Crypto; using Nethermind.Db.Blooms; using Nethermind.State.Repositories; +using Nethermind.TxPool; using Nethermind.Wallet; namespace Nethermind.Api { public interface IApiWithStores : IBasicApi { + ITxStorage? BlobTxStorage { get; set; } IBlockTree? BlockTree { get; set; } IBloomStorage? BloomStorage { get; set; } IChainLevelInfoRepository? ChainLevelInfoRepository { get; set; } diff --git a/src/Nethermind/Nethermind.Api/NethermindApi.cs b/src/Nethermind/Nethermind.Api/NethermindApi.cs index c94205abf91..86fe643e0c5 100644 --- a/src/Nethermind/Nethermind.Api/NethermindApi.cs +++ b/src/Nethermind/Nethermind.Api/NethermindApi.cs @@ -104,6 +104,7 @@ public IBlockchainBridge CreateBlockchainBridge() } public IAbiEncoder AbiEncoder { get; } = Nethermind.Abi.AbiEncoder.Instance; + public ITxStorage? BlobTxStorage { get; set; } public IBlockchainProcessor? BlockchainProcessor { get; set; } public CompositeBlockPreprocessorStep BlockPreprocessor { get; } = new(); public IBlockProcessingQueue? BlockProcessingQueue { get; set; } diff --git a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs index de14227e527..f23c2acfce7 100644 --- a/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs +++ b/src/Nethermind/Nethermind.Consensus.AuRa/InitializationSteps/InitializeBlockchainAuRa.cs @@ -274,7 +274,7 @@ protected override TxPool.TxPool CreateTxPool() return new TxPool.TxPool( _api.EthereumEcdsa, - new BlobTxStorage(_api.DbProvider.BlobTransactionsDb), + _api.BlobTxStorage ?? NullBlobTxStorage.Instance, new ChainHeadInfoProvider(_api.SpecProvider, _api.BlockTree, _api.StateReader), NethermindApi.Config(), _api.TxValidator, diff --git a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs index 179fbcd60ee..80358aed288 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs @@ -50,6 +50,9 @@ public async Task Execute(CancellationToken _) InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem); await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobDb); + _api.BlobTxStorage = useBlobDb + ? new BlobTxStorage(_api.DbProvider!.BlobTransactionsDb) + : NullBlobTxStorage.Instance; } catch (TypeInitializationException ex) { diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs index 358d03020a3..3e33d00de7a 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeBlockchain.cs @@ -333,7 +333,7 @@ protected virtual IHealthHintService CreateHealthHintService() => protected virtual TxPool.TxPool CreateTxPool() => new(_api.EthereumEcdsa!, - new BlobTxStorage(_api.DbProvider!.BlobTransactionsDb), + _api.BlobTxStorage ?? NullBlobTxStorage.Instance, new ChainHeadInfoProvider(_api.SpecProvider!, _api.BlockTree!, _api.StateReader!), _api.Config(), _api.TxValidator!, diff --git a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs index ceade1167cf..06d419ef487 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitializeNetwork.cs @@ -40,6 +40,7 @@ using Nethermind.Synchronization.Reporting; using Nethermind.Synchronization.SnapSync; using Nethermind.Synchronization.Trie; +using Nethermind.TxPool; namespace Nethermind.Init.Steps; @@ -528,7 +529,7 @@ private async Task InitPeer() ForkInfo forkInfo = new(_api.SpecProvider!, syncServer.Genesis.Hash!); ProtocolValidator protocolValidator = new(_api.NodeStatsManager!, _api.BlockTree, forkInfo, _api.LogManager); - PooledTxsRequestor pooledTxsRequestor = new(_api.TxPool!); + PooledTxsRequestor pooledTxsRequestor = new(_api.TxPool!, _api.Config()); _api.ProtocolsManager = new ProtocolsManager( _api.SyncPeerPool!, syncServer, diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs new file mode 100644 index 00000000000..b36ea34a964 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -0,0 +1,30 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using Nethermind.Core; +using Nethermind.Core.Crypto; + +namespace Nethermind.TxPool; + +public class NullBlobTxStorage : ITxStorage +{ + public static NullBlobTxStorage Instance { get; } = new(); + + public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) + { + transaction = default; + return false; + } + + public IEnumerable GetAll() + { + return ArraySegment.Empty; + } + + public void Add(Transaction transaction) { } + + public void Delete(ValueKeccak hash) { } +} From 8f6b3f42cb5dc6c50568d4e1da9ba094fa1e038f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 19:28:04 +0200 Subject: [PATCH 102/148] don't request blob txs if blob support disabled --- .../P2P/Subprotocols/Eth/IPooledTxsRequestor.cs | 2 +- .../P2P/Subprotocols/Eth/PooledTxsRequestor.cs | 15 +++++++++++---- .../Subprotocols/Eth/V68/Eth68ProtocolHandler.cs | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/IPooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/IPooledTxsRequestor.cs index 786186ebbf4..3feaaac63e0 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/IPooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/IPooledTxsRequestor.cs @@ -12,6 +12,6 @@ public interface IPooledTxsRequestor { void RequestTransactions(Action send, IReadOnlyList hashes); void RequestTransactionsEth66(Action send, IReadOnlyList hashes); - void RequestTransactionsEth68(Action send, IReadOnlyList hashes, IReadOnlyList sizes); + void RequestTransactionsEth68(Action send, IReadOnlyList hashes, IReadOnlyList sizes, IReadOnlyList types); } } diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs index 3dd79c22bbd..d14b9507733 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; +using Nethermind.Core; using Nethermind.Core.Caching; using Nethermind.Core.Collections; using Nethermind.Core.Crypto; @@ -16,12 +17,15 @@ public class PooledTxsRequestor : IPooledTxsRequestor { private const int MaxNumberOfTxsInOneMsg = 256; private readonly ITxPool _txPool; + private readonly ITxPoolConfig _txPoolConfig; + private readonly LruKeyCache _pendingHashes = new(MemoryAllowance.TxHashCacheSize, Math.Min(1024 * 16, MemoryAllowance.TxHashCacheSize), "pending tx hashes"); - public PooledTxsRequestor(ITxPool txPool) + public PooledTxsRequestor(ITxPool txPool, ITxPoolConfig txPoolConfig) { _txPool = txPool; + _txPoolConfig = txPoolConfig; } public void RequestTransactions(Action send, IReadOnlyList hashes) @@ -68,7 +72,7 @@ public void RequestTransactionsEth66(Action send, IReadOnlyList hashes, IReadOnlyList sizes) + public void RequestTransactionsEth68(Action send, IReadOnlyList hashes, IReadOnlyList sizes, IReadOnlyList types) { using ArrayPoolList<(Keccak Hash, int Size)> discoveredTxHashesAndSizes = new(hashes.Count); AddMarkUnknownHashesEth68(hashes, sizes, discoveredTxHashesAndSizes); @@ -89,8 +93,11 @@ public void RequestTransactionsEth68(Action 0) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs index 36d3547a2c9..2cab5def7fd 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs @@ -93,7 +93,7 @@ private void Handle(NewPooledTransactionHashesMessage68 message) Stopwatch? stopwatch = isTrace ? Stopwatch.StartNew() : null; - _pooledTxsRequestor.RequestTransactionsEth68(_sendAction, message.Hashes, message.Sizes); + _pooledTxsRequestor.RequestTransactionsEth68(_sendAction, message.Hashes, message.Sizes, message.Types); stopwatch?.Stop(); From b10053310899f647a598e1d0bf1b39588642601d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 19:37:00 +0200 Subject: [PATCH 103/148] fix tests --- .../Subprotocols/Eth/V65/PooledTxsRequestorTests.cs | 10 +++++----- .../Subprotocols/Eth/V66/Eth66ProtocolHandlerTests.cs | 2 +- .../Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTxsRequestorTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTxsRequestorTests.cs index 0bac0287522..5d42d59960d 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTxsRequestorTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V65/PooledTxsRequestorTests.cs @@ -29,7 +29,7 @@ public class PooledTxsRequestorTests public void filter_properly_newPooledTxHashes() { _response = new List(); - _requestor = new PooledTxsRequestor(_txPool); + _requestor = new PooledTxsRequestor(_txPool, new TxPoolConfig()); _requestor.RequestTransactions(_doNothing, new List { TestItem.KeccakA, TestItem.KeccakD }); _request = new List { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }; @@ -42,7 +42,7 @@ public void filter_properly_newPooledTxHashes() public void filter_properly_already_pending_hashes() { _response = new List(); - _requestor = new PooledTxsRequestor(_txPool); + _requestor = new PooledTxsRequestor(_txPool, new TxPoolConfig()); _requestor.RequestTransactions(_doNothing, new List { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }); _request = new List { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }; @@ -54,7 +54,7 @@ public void filter_properly_already_pending_hashes() public void filter_properly_discovered_hashes() { _response = new List(); - _requestor = new PooledTxsRequestor(_txPool); + _requestor = new PooledTxsRequestor(_txPool, new TxPoolConfig()); _request = new List { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }; _expected = new List { TestItem.KeccakA, TestItem.KeccakB, TestItem.KeccakC }; @@ -66,7 +66,7 @@ public void filter_properly_discovered_hashes() public void can_handle_empty_argument() { _response = new List(); - _requestor = new PooledTxsRequestor(_txPool); + _requestor = new PooledTxsRequestor(_txPool, new TxPoolConfig()); _requestor.RequestTransactions(Send, new List()); _response.Should().BeEmpty(); } @@ -77,7 +77,7 @@ public void filter_properly_hashes_present_in_hashCache() _response = new List(); ITxPool txPool = Substitute.For(); txPool.IsKnown(Arg.Any()).Returns(true); - _requestor = new PooledTxsRequestor(txPool); + _requestor = new PooledTxsRequestor(txPool, new TxPoolConfig()); _request = new List { TestItem.KeccakA, TestItem.KeccakB }; _expected = new List { }; diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandlerTests.cs index 4598b159ba6..e56f5af3e2b 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandlerTests.cs @@ -304,7 +304,7 @@ public void should_request_in_GetPooledTransactionsMessage_up_to_256_txs(int num new NodeStatsManager(_timerFactory, LimboLogs.Instance), _syncManager, _transactionPool, - new PooledTxsRequestor(_transactionPool), + new PooledTxsRequestor(_transactionPool, new TxPoolConfig()), _gossipPolicy, new ForkInfo(_specProvider, _genesisBlock.Header.Hash!), LimboLogs.Instance); diff --git a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs index a61f602f6d2..91b21fca318 100644 --- a/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs +++ b/src/Nethermind/Nethermind.Network.Test/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandlerTests.cs @@ -115,7 +115,7 @@ public void Can_handle_NewPooledTransactions_message([Values(0, 1, 2, 100)] int HandleZeroMessage(msg, Eth68MessageCode.NewPooledTransactionHashes); _pooledTxsRequestor.Received(canGossipTransactions ? 1 : 0).RequestTransactionsEth68(Arg.Any>(), - Arg.Any>(), Arg.Any>()); + Arg.Any>(), Arg.Any>(), Arg.Any>()); } [TestCase(true)] @@ -153,7 +153,7 @@ public void Should_process_huge_transaction() HandleZeroMessage(msg, Eth68MessageCode.NewPooledTransactionHashes); _pooledTxsRequestor.Received(1).RequestTransactionsEth68(Arg.Any>(), - Arg.Any>(), Arg.Any>()); + Arg.Any>(), Arg.Any>(), Arg.Any>()); } [TestCase(1)] @@ -202,7 +202,7 @@ public void should_divide_GetPooledTransactionsMessage_if_max_message_size_is_ex new NodeStatsManager(_timerFactory, LimboLogs.Instance), _syncManager, _transactionPool, - new PooledTxsRequestor(_transactionPool), + new PooledTxsRequestor(_transactionPool, new TxPoolConfig()), _gossipPolicy, new ForkInfo(_specProvider, _genesisBlock.Header.Hash!), LimboLogs.Instance, From 7de7fe6f9047a6f13366b9c557af7725ba6c0b6c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 19:38:33 +0200 Subject: [PATCH 104/148] fix files encoding --- .../Nethermind.TxPool/Filters/NotSupportedTxFilter.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs index d7191ac4338..b3e1f377096 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index b36ea34a964..040774e6c66 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System; From bbe23e3b722557f070f750a1c9319b818e3fa0bd Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 21:01:35 +0200 Subject: [PATCH 105/148] fix test --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs | 7 +++++-- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs index c2cf56efe07..aacc451c11c 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.cs @@ -1069,7 +1069,7 @@ public async Task should_notify_peer_only_once() txPoolPeer.Id.Returns(TestItem.PublicKeyA); _txPool.AddPeer(txPoolPeer); Transaction tx = AddTransactionToPool(); - await Task.Delay(500); + await Task.Delay(1000); txPoolPeer.Received(1).SendNewTransactions(Arg.Any>(), false); } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index 5f8b9c291ab..1d2aa73e188 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -113,9 +113,12 @@ private void BroadcastOnce(Transaction tx) } } - public void BroadcastOnce(ITxPoolPeer peer, Transaction[] txs) + public void AnnounceOnce(ITxPoolPeer peer, Transaction[] txs) { - Notify(peer, txs, false); + if (txs.Length > 0) + { + Notify(peer, txs, false); + } } public void BroadcastPersistentTxs() diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index bdea9c3c930..48ef07918ac 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -296,8 +296,8 @@ public void AddPeer(ITxPoolPeer peer) if (_broadcaster.AddPeer(peer)) { // worth to refactor and prepare tx snapshot in more efficient way - _broadcaster.BroadcastOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot()); - _broadcaster.BroadcastOnce(peer, _blobTransactionSnapshot ??= _blobTransactions.GetSnapshot()); + _broadcaster.AnnounceOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot()); + _broadcaster.AnnounceOnce(peer, _blobTransactionSnapshot ??= _blobTransactions.GetSnapshot()); if (_logger.IsTrace) _logger.Trace($"Added a peer to TX pool: {peer}"); } From cc33167c7886b67032680cb3e8ba4b74508c55c7 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 13 Sep 2023 21:43:44 +0200 Subject: [PATCH 106/148] cosmetics --- .../Eth62ProtocolHandlerBenchmarks.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs | 5 +---- src/Nethermind/Nethermind.TxPool/TxPool.cs | 7 +------ 3 files changed, 3 insertions(+), 11 deletions(-) diff --git a/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs b/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs index f874b7edcd4..7c65c42c584 100644 --- a/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs +++ b/src/Nethermind/Nethermind.Network.Benchmark/Eth62ProtocolHandlerBenchmarks.cs @@ -56,7 +56,7 @@ public void SetUp() var specProvider = MainnetSpecProvider.Instance; TxPool.TxPool txPool = new TxPool.TxPool( ecdsa, - new BlobTxStorage(new MemDb()), + new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(MainnetSpecProvider.Instance), tree, stateProvider), new TxPoolConfig(), new TxValidator(TestBlockchainIds.ChainId), diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 040774e6c66..99c307513c9 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -19,10 +19,7 @@ public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transa return false; } - public IEnumerable GetAll() - { - return ArraySegment.Empty; - } + public IEnumerable GetAll() => Array.Empty(); public void Add(Transaction transaction) { } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 48ef07918ac..87a1bac16fa 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -4,7 +4,6 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Channels; @@ -42,8 +41,6 @@ public class TxPool : ITxPool, IDisposable private readonly TxDistinctSortedPool _transactions; private readonly BlobTxDistinctSortedPool _blobTransactions; - private readonly ITxStorage _blobTxStorage; - private readonly IChainHeadSpecProvider _specProvider; private readonly IAccountStateProvider _accounts; private readonly IChainHeadInfoProvider _headInfo; @@ -90,7 +87,6 @@ public TxPool(IEthereumEcdsa ecdsa, bool thereIsPriorityContract = false) { _logger = logManager?.GetClassLogger() ?? throw new ArgumentNullException(nameof(logManager)); - _blobTxStorage = blobTxStorage ?? throw new ArgumentNullException(nameof(blobTxStorage)); _headInfo = chainHeadInfoProvider ?? throw new ArgumentNullException(nameof(chainHeadInfoProvider)); _txPoolConfig = txPoolConfig; _accounts = _headInfo.AccountStateProvider; @@ -101,7 +97,7 @@ public TxPool(IEthereumEcdsa ecdsa, _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); _blobTransactions = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true } - ? new PersistentBlobTxDistinctSortedPool(_blobTxStorage, _txPoolConfig, comparer, logManager) + ? new PersistentBlobTxDistinctSortedPool(blobTxStorage, _txPoolConfig, comparer, logManager) : new BlobTxDistinctSortedPool(txPoolConfig.BlobSupportEnabled ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); @@ -295,7 +291,6 @@ public void AddPeer(ITxPoolPeer peer) { if (_broadcaster.AddPeer(peer)) { - // worth to refactor and prepare tx snapshot in more efficient way _broadcaster.AnnounceOnce(peer, _transactionSnapshot ??= _transactions.GetSnapshot()); _broadcaster.AnnounceOnce(peer, _blobTransactionSnapshot ??= _blobTransactions.GetSnapshot()); From 8173c81de98a3bfdd9654c195aad2ec491467ee2 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 14 Sep 2023 20:08:52 +0200 Subject: [PATCH 107/148] add MaxPendingTxsPerSenderFilter --- .../Nethermind.TxPool/AcceptTxResult.cs | 9 +++- .../Filters/MaxPendingTxsPerSenderFilter.cs | 43 +++++++++++++++++++ .../Nethermind.TxPool/ITxPoolConfig.cs | 6 +++ src/Nethermind/Nethermind.TxPool/Metrics.cs | 4 ++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 1 + .../Nethermind.TxPool/TxPoolConfig.cs | 2 + 6 files changed, 63 insertions(+), 2 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index fa3bca1d683..04a5fec68bf 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -75,15 +75,20 @@ namespace Nethermind.TxPool /// public static readonly AcceptTxResult SenderIsContract = new(12, nameof(SenderIsContract)); + /// + /// The nonce is too far in the future. + /// + public static readonly AcceptTxResult NonceTooFarInFuture = new(13, nameof(NonceTooFarInFuture)); + /// /// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs /// - public static readonly AcceptTxResult PendingTxsOfOtherType = new(13, nameof(PendingTxsOfOtherType)); + public static readonly AcceptTxResult PendingTxsOfOtherType = new(14, nameof(PendingTxsOfOtherType)); /// /// Ignores transactions if tx type is not supported /// - public static readonly AcceptTxResult NotSupportedTxType = new(14, nameof(NotSupportedTxType)); + public static readonly AcceptTxResult NotSupportedTxType = new(15, nameof(NotSupportedTxType)); private int Id { get; } private string Code { get; } diff --git a/src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs new file mode 100644 index 00000000000..053914b5caa --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs @@ -0,0 +1,43 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.TxPool.Collections; + +namespace Nethermind.TxPool.Filters; + +public class MaxPendingTxsPerSenderFilter : IIncomingTxFilter +{ + private readonly ITxPoolConfig _txPoolConfig; + private readonly TxDistinctSortedPool _txs; + private readonly TxDistinctSortedPool _blobTxs; + + public MaxPendingTxsPerSenderFilter(ITxPoolConfig txPoolConfig, TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) + { + _txPoolConfig = txPoolConfig; + _txs = txs; + _blobTxs = blobTxs; + } + + public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) + { + int relevantMaxPendingTxsPerSender = (tx.SupportsBlobs + ? _txPoolConfig.MaxPendingBlobTxsPerSender + : _txPoolConfig.MaxPendingTxsPerSender); + + if (relevantMaxPendingTxsPerSender == 0) + { + return AcceptTxResult.Accepted; + } + + TxDistinctSortedPool relevantTxPool = (tx.SupportsBlobs ? _blobTxs : _txs); + + if (relevantTxPool.GetBucketCount(tx.SenderAddress!) > relevantMaxPendingTxsPerSender) + { + Metrics.PendingTransactionsNonceTooFarInFuture++; + return AcceptTxResult.NonceTooFarInFuture; + } + + return AcceptTxResult.Accepted; + } +} diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index a8c64d521c1..768a1c53d4c 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -28,6 +28,12 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "512", Description = "Max number of full blob transactions stored in memory. Used only if persistent storage is disabled")] int InMemoryBlobPoolSize { get; set; } + [ConfigItem(DefaultValue = "0", Description = "Max number of pending transactions per single sender. Infinite by default (0)")] + int MaxPendingTxsPerSender { get; set; } + + [ConfigItem(DefaultValue = "16", Description = "Max number of pending blob transactions per single sender.")] + int MaxPendingBlobTxsPerSender { get; set; } + [ConfigItem(DefaultValue = "524288", Description = "Max number of cached hashes of already known transactions." + "It is set automatically by the memory hint.")] diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 49abd5fd053..4e7f2bc50ce 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -83,6 +83,10 @@ public static class Metrics [Description("Number of transactions with already used nonce.")] public static long PendingTransactionsLowNonce { get; set; } + [CounterMetric] + [Description("Number of transactions with nonce too far in future.")] + public static long PendingTransactionsNonceTooFarInFuture { get; set; } + [CounterMetric] [Description("Number of transactions rejected because of already pending tx of other type (allowed blob txs or others, not both at once).")] public static long PendingTransactionsConflictingTxType { get; set; } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 87a1bac16fa..e776a275a90 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -117,6 +117,7 @@ public TxPool(IEthereumEcdsa ecdsa, new AlreadyKnownTxFilter(_hashCache, _logger), new UnknownSenderFilter(ecdsa, _logger), new TxTypeTxFilter(_transactions, _blobTransactions), // has to be after UnknownSenderFilter as it uses sender + new MaxPendingTxsPerSenderFilter(txPoolConfig, _transactions, _blobTransactions), new BalanceZeroFilter(thereIsPriorityContract, _logger), new BalanceTooLowFilter(_transactions, _blobTransactions, _logger), new LowNonceFilter(_logger), // has to be after UnknownSenderFilter as it uses sender diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index 2ca2fb5ddb0..b9e983b8053 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -14,6 +14,8 @@ public class TxPoolConfig : ITxPoolConfig // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) public int BlobCacheSize { get; set; } = 256; public int InMemoryBlobPoolSize { get; set; } = 512; // it is used when persistent pool is disabled + public int MaxPendingTxsPerSender { get; set; } = 0; + public int MaxPendingBlobTxsPerSender { get; set; } = 16; public int HashCacheSize { get; set; } = 512 * 1024; public long? GasLimit { get; set; } = null; public int? ReportMinutes { get; set; } = null; From 6195466be61d49d7a5baafabf57f20f93de0260d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 14 Sep 2023 20:27:53 +0200 Subject: [PATCH 108/148] add test --- .../TxPoolTests.Blobs.cs | 35 +++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index be17af9b2ec..33bd15f8766 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -1,13 +1,11 @@ // SPDX-FileCopyrightText: 2022 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only -using System.Linq; using System.Threading.Tasks; using FluentAssertions; using Nethermind.Core; using Nethermind.Core.Extensions; using Nethermind.Core.Test.Builders; -using Nethermind.Db; using Nethermind.Evm; using Nethermind.Int256; using NUnit.Framework; @@ -65,6 +63,39 @@ public void blob_pool_size_should_be_correct([Values(true, false)] bool persiste _txPool.GetPendingBlobTransactionsCount().Should().Be(poolSize); } + [TestCase(TxType.EIP1559, 0, 5, 100)] + [TestCase(TxType.Blob, 5, 0, 100)] + [TestCase(TxType.EIP1559, 10, 0, 10)] + [TestCase(TxType.Blob, 0, 15, 15)] + [TestCase(TxType.EIP1559, 20, 25, 20)] + [TestCase(TxType.Blob, 30, 35, 35)] + public void should_reject_txs_with_nonce_too_far_in_future(TxType txType, int maxPendingTxs, int maxPendingBlobTxs, int expectedNumberOfAcceptedTxs) + { + TxPoolConfig txPoolConfig = new() + { + Size = 100, + MaxPendingTxsPerSender = maxPendingTxs, + MaxPendingBlobTxsPerSender = maxPendingBlobTxs + }; + + _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); + EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); + for (int nonce = 0; nonce < txPoolConfig.Size; nonce++) + { + Transaction tx = Build.A.Transaction + .WithNonce((UInt256)nonce) + .WithType(txType) + .WithShardBlobTxTypeAndFieldsIfBlobTx() + .WithMaxFeePerGas(1.GWei()) + .WithMaxPriorityFeePerGas(1.GWei()) + .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; + + _txPool.SubmitTx(tx, TxHandlingOptions.None).Should().Be(nonce > expectedNumberOfAcceptedTxs + ? AcceptTxResult.NonceTooFarInFuture + : AcceptTxResult.Accepted); + } + } + [Test] public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, false)] bool isBlob, [Values(true, false)] bool persistentStorageEnabled) { From ee1891b761808a98b3e9edac42866ec2b6947cad Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Sep 2023 12:47:24 +0200 Subject: [PATCH 109/148] disable blob pool by default --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 768a1c53d4c..2de0cf2d57b 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -13,7 +13,7 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "2048", Description = "Max number of transactions held in mempool (more transactions in mempool mean more memory used")] int Size { get; set; } - [ConfigItem(DefaultValue = "true", Description = "If true, blob transactions support will be enabled")] + [ConfigItem(DefaultValue = "false", Description = "If true, blob transactions support will be enabled")] bool BlobSupportEnabled { get; set; } [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index b9e983b8053..e6975cbadb2 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -7,7 +7,7 @@ public class TxPoolConfig : ITxPoolConfig { public int PeerNotificationThreshold { get; set; } = 5; public int Size { get; set; } = 2048; - public bool BlobSupportEnabled { get; set; } = true; + public bool BlobSupportEnabled { get; set; } = false; public bool PersistentBlobStorageEnabled { get; set; } = false; public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 12,5GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. From d2131bdd7987834bb6148dd06c79844e0b0856d5 Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Tue, 19 Sep 2023 13:11:59 +0200 Subject: [PATCH 110/148] cosmetic refactoring --- .../Collections/BlobTxDistinctSortedPool.cs | 6 ------ src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- src/tests | 2 +- 3 files changed, 2 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index f1e3b49b719..8ddab60c89a 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -27,10 +27,4 @@ public override void EnsureCapacity() if (Count > _poolCapacity && _logger.IsWarn) _logger.Warn($"Blob pool exceeds the config size {Count}/{_poolCapacity}"); } - - /// - /// For tests only - to test sorting - /// - internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? lightBlobTx) - => base.TryGetValue(hash, out lightBlobTx); } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e776a275a90..90615dc64c6 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -610,7 +610,7 @@ public bool TryGetPendingBlobTransaction(Keccak hash, [NotNullWhen(true)] out Tr // only for tests - to test sorting internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? transaction) - => _blobTransactions.TryGetBlobTxSortingEquivalent(hash, out transaction); + => _blobTransactions.TryGetValue(hash, out transaction); // should own transactions (in broadcaster) be also checked here? // maybe it should use NonceManager, as it already has info about local txs? diff --git a/src/tests b/src/tests index 9b00b68593f..661356317ac 160000 --- a/src/tests +++ b/src/tests @@ -1 +1 @@ -Subproject commit 9b00b68593f5869eb51a6659e1cc983e875e616b +Subproject commit 661356317ac6df52208d54187e692472a25a01f8 From ae671d451dcd2ec5e44a5c263aa432473696a2ee Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Tue, 19 Sep 2023 13:21:14 +0200 Subject: [PATCH 111/148] revert previous change --- .../Collections/BlobTxDistinctSortedPool.cs | 6 ++++++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index 8ddab60c89a..f1e3b49b719 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -27,4 +27,10 @@ public override void EnsureCapacity() if (Count > _poolCapacity && _logger.IsWarn) _logger.Warn($"Blob pool exceeds the config size {Count}/{_poolCapacity}"); } + + /// + /// For tests only - to test sorting + /// + internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? lightBlobTx) + => base.TryGetValue(hash, out lightBlobTx); } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 90615dc64c6..e776a275a90 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -610,7 +610,7 @@ public bool TryGetPendingBlobTransaction(Keccak hash, [NotNullWhen(true)] out Tr // only for tests - to test sorting internal void TryGetBlobTxSortingEquivalent(Keccak hash, out Transaction? transaction) - => _blobTransactions.TryGetValue(hash, out transaction); + => _blobTransactions.TryGetBlobTxSortingEquivalent(hash, out transaction); // should own transactions (in broadcaster) be also checked here? // maybe it should use NonceManager, as it already has info about local txs? From ccc1799e600ed95d77d2aa7d55f338899719918c Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Sep 2023 13:56:05 +0200 Subject: [PATCH 112/148] fix tests after making blob pool disabled by default --- .../Blockchain/TestBlockchain.cs | 2 +- .../TxPoolTests.Blobs.cs | 45 ++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs index 9bb0026bda2..77ddd57c33d 100644 --- a/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs +++ b/src/Nethermind/Nethermind.Core.Test/Blockchain/TestBlockchain.cs @@ -280,7 +280,7 @@ protected virtual TxPool.TxPool CreateTxPool() => EthereumEcdsa, new BlobTxStorage(), new ChainHeadInfoProvider(new FixedForkActivationChainHeadSpecProvider(SpecProvider), BlockTree, ReadOnlyState), - new TxPoolConfig(), + new TxPoolConfig() { BlobSupportEnabled = true }, new TxValidator(SpecProvider.ChainId), LogManager, TransactionComparerProvider.GetDefaultComparer()); diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 33bd15f8766..be6f1118990 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -40,11 +40,11 @@ public void blob_pool_size_should_be_correct([Values(true, false)] bool persiste const int poolSize = 10; TxPoolConfig txPoolConfig = new() { + BlobSupportEnabled = true, PersistentBlobStorageEnabled = persistentStorageEnabled, PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, InMemoryBlobPoolSize = persistentStorageEnabled ? 0 : poolSize }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -73,11 +73,11 @@ public void should_reject_txs_with_nonce_too_far_in_future(TxType txType, int ma { TxPoolConfig txPoolConfig = new() { + BlobSupportEnabled = true, Size = 100, MaxPendingTxsPerSender = maxPendingTxs, MaxPendingBlobTxsPerSender = maxPendingBlobTxs }; - _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); for (int nonce = 0; nonce < txPoolConfig.Size; nonce++) @@ -102,6 +102,7 @@ public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, f const int poolSize = 10; TxPoolConfig txPoolConfig = new() { + BlobSupportEnabled = true, Size = isBlob ? 0 : poolSize, PersistentBlobStorageEnabled = persistentStorageEnabled, PersistentBlobStorageSize = persistentStorageEnabled ? poolSize : 0, @@ -141,7 +142,12 @@ public void should_reject_tx_with_FeeTooLow_even_if_is_blob_type([Values(true, f [Test] public void should_add_blob_tx_and_return_when_requested([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() + { + BlobSupportEnabled = true, + Size = 10, + PersistentBlobStorageEnabled = isPersistentStorage + }; BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -186,7 +192,7 @@ public void should_not_throw_when_asking_for_non_existing_tx() [TestCase(1_000_000_000, true)] public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than_1GWei(int maxPriorityFeePerGas, bool expectedResult) { - TxPoolConfig txPoolConfig = new() { Size = 10 }; + TxPoolConfig txPoolConfig = new() { BlobSupportEnabled = true, Size = 10 }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -204,7 +210,7 @@ public void should_not_allow_to_add_blob_tx_with_MaxPriorityFeePerGas_lower_than [Test] public void should_not_add_nonce_gap_blob_tx_even_to_not_full_TxPool([Values(true, false)] bool isBlob) { - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction @@ -241,7 +247,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); @@ -254,7 +260,12 @@ Transaction GetTx(bool isBlob, UInt256 nonce) [Test] public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = true }; + TxPoolConfig txPoolConfig = new() + { + BlobSupportEnabled = true, + Size = 10, + PersistentBlobStorageEnabled = true + }; BlobTxStorage blobTxStorage = new(); _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider(), txStorage: blobTxStorage); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -300,7 +311,12 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() [Test] public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_storage_enabled([Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() + { + BlobSupportEnabled = true, + Size = 10, + PersistentBlobStorageEnabled = isPersistentStorage + }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -323,7 +339,12 @@ public void should_keep_in_memory_only_light_blob_tx_equivalent_if_persistent_st [Test] public void should_dump_GasBottleneck_of_blob_tx_to_zero_if_MaxFeePerBlobGas_is_lower_than_current([Values(true, false)] bool isBlob, [Values(true, false)] bool isPersistentStorage) { - TxPoolConfig txPoolConfig = new() { Size = 10, PersistentBlobStorageEnabled = isPersistentStorage }; + TxPoolConfig txPoolConfig = new() + { + BlobSupportEnabled = true, + Size = 10, + PersistentBlobStorageEnabled = isPersistentStorage + }; _txPool = CreatePool(txPoolConfig, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -362,7 +383,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, { bool shouldReplace = blobsInFirstTx <= blobsInSecondTx; - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = Build.A.Transaction @@ -395,7 +416,7 @@ public void should_not_allow_to_replace_blob_tx_by_tx_with_less_blobs([Values(1, [Test] public void should_discard_tx_when_data_gas_cost_cause_overflow([Values(false, true)] bool supportsBlobs) { - _txPool = CreatePool(null, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); @@ -437,7 +458,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) .SignedAndResolved(_ethereumEcdsa, TestItem.PrivateKeyA).TestObject; } - _txPool = CreatePool(new TxPoolConfig() { Size = 128 }, GetCancunSpecProvider()); + _txPool = CreatePool(new TxPoolConfig() { BlobSupportEnabled = true, Size = 128 }, GetCancunSpecProvider()); EnsureSenderBalance(TestItem.AddressA, UInt256.MaxValue); Transaction firstTx = GetTx(firstIsBlob, UInt256.Zero); From 0bb5cf0a1482eea7341e57d64bd649240ac4dbd0 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 19 Sep 2023 14:02:23 +0200 Subject: [PATCH 113/148] useBlobDb -> useBlobsDb --- .../Nethermind.Db/StandardDbInitializer.cs | 12 ++++++------ src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs | 6 +++--- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index 60298e95abc..257d0cab5a9 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -23,19 +23,19 @@ public StandardDbInitializer( _fileSystem = fileSystem ?? new FileSystem(); } - public void InitStandardDbs(bool useReceiptsDb, bool useBlobDb = true) + public void InitStandardDbs(bool useReceiptsDb, bool useBlobsDb = true) { - RegisterAll(useReceiptsDb, useBlobDb); + RegisterAll(useReceiptsDb, useBlobsDb); InitAll(); } - public async Task InitStandardDbsAsync(bool useReceiptsDb, bool useBlobDb = true) + public async Task InitStandardDbsAsync(bool useReceiptsDb, bool useBlobsDb = true) { - RegisterAll(useReceiptsDb, useBlobDb); + RegisterAll(useReceiptsDb, useBlobsDb); await InitAllAsync(); } - private void RegisterAll(bool useReceiptsDb, bool useBlobDb) + private void RegisterAll(bool useReceiptsDb, bool useBlobsDb) { RegisterDb(BuildRocksDbSettings(DbNames.Blocks, () => Metrics.BlocksDbReads++, () => Metrics.BlocksDbWrites++)); RegisterDb(BuildRocksDbSettings(DbNames.Headers, () => Metrics.HeaderDbReads++, () => Metrics.HeaderDbWrites++)); @@ -62,7 +62,7 @@ private void RegisterAll(bool useReceiptsDb, bool useBlobDb) RegisterCustomDb(DbNames.Receipts, () => new ReadOnlyColumnsDb(new MemColumnsDb(), false)); } RegisterDb(BuildRocksDbSettings(DbNames.Metadata, () => Metrics.MetadataDbReads++, () => Metrics.MetadataDbWrites++)); - if (useBlobDb) + if (useBlobsDb) { RegisterDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); } diff --git a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs index 80358aed288..cd1de616fc2 100644 --- a/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs +++ b/src/Nethermind/Nethermind.Init/Steps/InitDatabase.cs @@ -46,11 +46,11 @@ public async Task Execute(CancellationToken _) try { bool useReceiptsDb = initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync; - bool useBlobDb = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true }; + bool useBlobsDb = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true }; InitDbApi(initConfig, dbConfig, initConfig.StoreReceipts || syncConfig.DownloadReceiptsInFastSync); StandardDbInitializer dbInitializer = new(_api.DbProvider, _api.RocksDbFactory, _api.MemDbFactory, _api.FileSystem); - await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobDb); - _api.BlobTxStorage = useBlobDb + await dbInitializer.InitStandardDbsAsync(useReceiptsDb, useBlobsDb); + _api.BlobTxStorage = useBlobsDb ? new BlobTxStorage(_api.DbProvider!.BlobTransactionsDb) : NullBlobTxStorage.Instance; } From 3040dfa1daaa35a0948b46666b8b277c356dc206 Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Tue, 19 Sep 2023 18:26:15 +0200 Subject: [PATCH 114/148] TxConfig descrptions fixes --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 2de0cf2d57b..4e3dcd52bc1 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -19,7 +19,7 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] bool PersistentBlobStorageEnabled { get; set; } - [ConfigItem(DefaultValue = "16384", Description = "Max number of full blob transactions stored in db (but more transactions in blob pool mean more memory use too")] + [ConfigItem(DefaultValue = "16384", Description = "Max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage)")] int PersistentBlobStorageSize { get; set; } [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage")] @@ -28,10 +28,10 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "512", Description = "Max number of full blob transactions stored in memory. Used only if persistent storage is disabled")] int InMemoryBlobPoolSize { get; set; } - [ConfigItem(DefaultValue = "0", Description = "Max number of pending transactions per single sender. Infinite by default (0)")] + [ConfigItem(DefaultValue = "0", Description = "Max number of pending transactions per single sender. Set it to 0 to disable the limit.")] int MaxPendingTxsPerSender { get; set; } - [ConfigItem(DefaultValue = "16", Description = "Max number of pending blob transactions per single sender.")] + [ConfigItem(DefaultValue = "16", Description = "Max number of pending blob transactions per single sender. Set it to 0 to disable the limit.")] int MaxPendingBlobTxsPerSender { get; set; } [ConfigItem(DefaultValue = "524288", From 84e8e79540b6b75ee3befc18ff72e567cb8964d4 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 18 Sep 2023 16:56:52 +0200 Subject: [PATCH 115/148] add timestamp to db, drop linq --- .../Nethermind.TxPool/BlobTxStorage.cs | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 52ea91216fa..abdd7cf6373 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -4,11 +4,10 @@ using System; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; -using System.Linq; using Nethermind.Core; using Nethermind.Core.Crypto; -using Nethermind.Core.Extensions; using Nethermind.Db; +using Nethermind.Int256; using Nethermind.Serialization.Rlp; namespace Nethermind.TxPool; @@ -16,6 +15,7 @@ namespace Nethermind.TxPool; public class BlobTxStorage : ITxStorage { private readonly IDb _database; + private static readonly TxDecoder _txDecoder = new(); public BlobTxStorage() { @@ -35,9 +35,11 @@ private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) if (txBytes is not null) { RlpStream rlpStream = new(txBytes); - Address sender = new(rlpStream.Read(20).ToArray()); + Address sender = rlpStream.DecodeAddress()!; + UInt256 timestamp = rlpStream.DecodeUInt256(); transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); transaction.SenderAddress = sender; + transaction.Timestamp = timestamp; return true; } @@ -63,11 +65,16 @@ public void Add(Transaction transaction) throw new ArgumentNullException(nameof(transaction)); } - _database.Set(transaction.Hash, - Bytes.Concat( - transaction.SenderAddress!.Bytes, - Rlp.Encode(transaction, RlpBehaviors.InMempoolForm).Bytes - ).ToArray()); + int length = Rlp.LengthOf(transaction.SenderAddress); + length += Rlp.LengthOf(transaction.Timestamp); + length += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + + RlpStream rlpStream = new(length); + rlpStream.Encode(transaction.SenderAddress); + rlpStream.Encode(transaction.Timestamp); + rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); + + _database.Set(transaction.Hash, rlpStream.Data!); } public void Delete(ValueKeccak hash) From 75163e7a793d94fa3bdbf6db1081cd8969b07b7e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Sep 2023 19:31:03 +0200 Subject: [PATCH 116/148] add missing metric --- .../Nethermind.TxPool/Filters/NotSupportedTxFilter.cs | 1 + src/Nethermind/Nethermind.TxPool/Metrics.cs | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs index b3e1f377096..2f2e264e3d6 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/NotSupportedTxFilter.cs @@ -24,6 +24,7 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO { if (!_txPoolConfig.BlobSupportEnabled && tx.SupportsBlobs) { + Metrics.PendingTransactionsNotSupportedTxType++; if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, blob transactions are not supported."); return AcceptTxResult.NotSupportedTxType; } diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 4e7f2bc50ce..96c4daf69d7 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -24,6 +24,10 @@ public static class Metrics [Description("Number of pending transactions received that were ignored.")] public static long PendingTransactionsDiscarded { get; set; } + [CounterMetric] + [Description("Number of pending transactions received that were ignored because of not supported transaction type.")] + public static long PendingTransactionsNotSupportedTxType { get; set; } + [CounterMetric] [Description( "Number of pending transactions received that were ignored because of not having preceding nonce of this sender in TxPool.")] From 8f9980ef3d54fbfee663bf512b1c982ed3dd74c4 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Sep 2023 19:33:02 +0200 Subject: [PATCH 117/148] adjust txpool report --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 30 +++++++++++++--------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index e776a275a90..a8c70d5caa8 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -720,19 +720,21 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) Total Received: {Metrics.PendingTransactionsReceived,24:N0} ------------------------------------------------ Discarded at Filter Stage: -1. GasLimitTooHigh: {Metrics.PendingTransactionsGasLimitTooHigh,24:N0} -2. Too Low Fee: {Metrics.PendingTransactionsTooLowFee,24:N0} -3. Malformed {Metrics.PendingTransactionsMalformed,24:N0} -4. Duplicate: {Metrics.PendingTransactionsKnown,24:N0} -5. Unknown Sender: {Metrics.PendingTransactionsUnresolvableSender,24:N0} -6. Conflicting TxType {Metrics.PendingTransactionsConflictingTxType,24:N0} -7. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} -8. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} -9. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} -10. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} +1. NotSupportedTxType {Metrics.PendingTransactionsNotSupportedTxType,24:N0} +2. GasLimitTooHigh: {Metrics.PendingTransactionsGasLimitTooHigh,24:N0} +3. Too Low Fee: {Metrics.PendingTransactionsTooLowFee,24:N0} +4. Malformed {Metrics.PendingTransactionsMalformed,24:N0} +5. Duplicate: {Metrics.PendingTransactionsKnown,24:N0} +6. Unknown Sender: {Metrics.PendingTransactionsUnresolvableSender,24:N0} +7. Conflicting TxType {Metrics.PendingTransactionsConflictingTxType,24:N0} +8. NonceTooFarInFuture {Metrics.PendingTransactionsNonceTooFarInFuture,24:N0} +9. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} +10. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} 11. Balance Too Low: {Metrics.PendingTransactionsTooLowBalance,24:N0} -12. Failed replacement {Metrics.PendingTransactionsPassedFiltersButCannotReplace,24:N0} -13. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} +12. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} +13. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} +14. Failed replacement {Metrics.PendingTransactionsPassedFiltersButCannotReplace,24:N0} +15. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} ------------------------------------------------ Validated via State: {Metrics.PendingTransactionsWithExpensiveFiltering,24:N0} ------------------------------------------------ @@ -756,6 +758,10 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) * Blob txs: {Metrics.BlobTransactionsInBlock,24:N0} * Blobs: {Metrics.BlobsInBlock,24:N0} ------------------------------------------------ +Db usage: +* BlobDb writes: {Db.Metrics.BlobTransactionsDbWrites,24:N0} +* BlobDb reads: {Db.Metrics.BlobTransactionsDbReads,24:N0} +------------------------------------------------ "); } } From ce6be6c72542adf525cc3e6ca048d33292937e4f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 20 Sep 2023 19:52:41 +0200 Subject: [PATCH 118/148] cosmetic --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index a8c70d5caa8..cb71dcc7a2d 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -746,7 +746,7 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) ------------------------------------------------ Total Added: {Metrics.PendingTransactionsAdded,24:N0} * Eip1559 Added: {Metrics.Pending1559TransactionsAdded,24:N0} -* Blob Added: {Metrics.PendingBlobTransactionsAdded,24:N0} +* Blob Added: {Metrics.PendingBlobTransactionsAdded,24:N0} ------------------------------------------------ Total Evicted: {Metrics.PendingTransactionsEvicted,24:N0} ------------------------------------------------ From 0ade121a49211f4800179cbb2a67f06001f6a08a Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Sep 2023 11:38:50 +0200 Subject: [PATCH 119/148] fix broadcaster to check in pending before requesting --- .../P2P/Subprotocols/Eth/PooledTxsRequestor.cs | 13 +++++++------ .../Nethermind.TxPool/Collections/SortedPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 1 + src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs | 2 ++ src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++++ 5 files changed, 15 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs index d14b9507733..a882bbf3b32 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/PooledTxsRequestor.cs @@ -74,8 +74,8 @@ public void RequestTransactionsEth66(Action send, IReadOnlyList hashes, IReadOnlyList sizes, IReadOnlyList types) { - using ArrayPoolList<(Keccak Hash, int Size)> discoveredTxHashesAndSizes = new(hashes.Count); - AddMarkUnknownHashesEth68(hashes, sizes, discoveredTxHashesAndSizes); + using ArrayPoolList<(Keccak Hash, byte Type, int Size)> discoveredTxHashesAndSizes = new(hashes.Count); + AddMarkUnknownHashesEth68(hashes, sizes, types, discoveredTxHashesAndSizes); if (discoveredTxHashesAndSizes.Count != 0) { @@ -85,6 +85,7 @@ public void RequestTransactionsEth68(Action packetSizeLeft && hashesToRequest.Count > 0) { @@ -93,7 +94,7 @@ public void RequestTransactionsEth68(Action hashes, ArrayPoolList hashes, IReadOnlyList sizes, ArrayPoolList<(Keccak, int)> discoveredTxHashesAndSizes) + private void AddMarkUnknownHashesEth68(IReadOnlyList hashes, IReadOnlyList sizes, IReadOnlyList types, ArrayPoolList<(Keccak, byte, int)> discoveredTxHashesAndSizes) { int count = hashes.Count; for (int i = 0; i < count; i++) { Keccak hash = hashes[i]; - if (!_txPool.IsKnown(hash) && _pendingHashes.Set(hash)) + if (!_txPool.IsKnown(hash) && !_txPool.ContainsTx(hash, (TxType)types[i]) && _pendingHashes.Set(hash)) { - discoveredTxHashesAndSizes.Add((hash, sizes[i])); + discoveredTxHashesAndSizes.Add((hash, types[i], sizes[i])); } } } diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index a3ef472ad8b..4420b836e55 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -261,7 +261,7 @@ public IEnumerable TakeWhile(TGroupKey groupKey, Predicate where /// Key to check presence. /// True if element is present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - protected bool ContainsValue(TKey key) + public bool ContainsValue(TKey key) { return _cacheMap.ContainsKey(key); } diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index e03cdb85a9f..f3392ee6ae4 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -35,6 +35,7 @@ public interface ITxPool Transaction[] GetPendingTransactionsBySender(Address address); void AddPeer(ITxPoolPeer peer); void RemovePeer(PublicKey nodeId); + bool ContainsTx(Keccak hash, TxType txType); AcceptTxResult SubmitTx(Transaction tx, TxHandlingOptions handlingOptions); bool RemoveTransaction(Keccak? hash); bool IsKnown(Keccak hash); diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index 1d2aa73e188..b316bd7ba13 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -315,6 +315,8 @@ public bool TryGetPersistentTx(Keccak hash, out Transaction? transaction) return false; } + public bool ContainsTx(Keccak hash) => _persistentTxs.ContainsValue(hash); + public bool AddPeer(ITxPoolPeer peer) { return _peers.TryAdd(peer.Id, peer); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index cb71dcc7a2d..38c5f488059 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -590,6 +590,10 @@ public bool RemoveTransaction(Keccak? hash) return hasBeenRemoved; } + public bool ContainsTx(Keccak hash, TxType txType) => txType == TxType.Blob + ? _blobTransactions.ContainsValue(hash) + : _transactions.ContainsValue(hash) || _broadcaster.ContainsTx(hash); + public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) { lock (_locker) From a4d3600dace61750040a5a016d57a6bc657fb61e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Sep 2023 14:31:12 +0200 Subject: [PATCH 120/148] fix --- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 52ef615a984..7944ea06cd8 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -34,6 +34,8 @@ public void AddPeer(ITxPoolPeer peer) { } public void RemovePeer(PublicKey nodeId) { } + public bool ContainsTx(Keccak hash, TxType txType) => false; + public AcceptTxResult SubmitTx(Transaction tx, TxHandlingOptions txHandlingOptions) => AcceptTxResult.Accepted; public bool RemoveTransaction(Keccak? hash) => false; From 918c7b6fad7358eb8fcf78bc1dc785e3df1cca37 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Sep 2023 16:02:39 +0200 Subject: [PATCH 121/148] add metric of received hashes --- .../P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs | 1 + .../P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs | 1 + .../P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs | 1 + src/Nethermind/Nethermind.TxPool/Metrics.cs | 4 ++++ 4 files changed, 7 insertions(+) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs index 4ede85c04e1..265c83dba0f 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs @@ -100,6 +100,7 @@ protected virtual void Handle(NewPooledTransactionHashesMessage msg) Stopwatch stopwatch = Stopwatch.StartNew(); + TxPool.Metrics.PendingTransactionsHashesReceived += msg.Hashes.Count; _pooledTxsRequestor.RequestTransactions(Send, msg.Hashes); stopwatch.Stop(); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs index cc55d85827c..93f5f310b22 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V66/Eth66ProtocolHandler.cs @@ -205,6 +205,7 @@ protected override void Handle(NewPooledTransactionHashesMessage msg) bool isTrace = Logger.IsTrace; Stopwatch? stopwatch = isTrace ? Stopwatch.StartNew() : null; + TxPool.Metrics.PendingTransactionsHashesReceived += msg.Hashes.Count; _pooledTxsRequestor.RequestTransactionsEth66(_sendAction, msg.Hashes); stopwatch?.Stop(); diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs index 2cab5def7fd..38ba9ce84c5 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V68/Eth68ProtocolHandler.cs @@ -88,6 +88,7 @@ private void Handle(NewPooledTransactionHashesMessage68 message) } Metrics.Eth68NewPooledTransactionHashesReceived++; + TxPool.Metrics.PendingTransactionsHashesReceived += message.Hashes.Count; AddNotifiedTransactions(message.Hashes); diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 96c4daf69d7..1668bd08099 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -20,6 +20,10 @@ public static class Metrics [Description("Number of pending transactions received from peers.")] public static long PendingTransactionsReceived { get; set; } + [CounterMetric] + [Description("Number of hashes of pending transactions received from peers.")] + public static long PendingTransactionsHashesReceived { get; set; } + [CounterMetric] [Description("Number of pending transactions received that were ignored.")] public static long PendingTransactionsDiscarded { get; set; } From 3b63ce5611691ca537de0492048373cf03adceff Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 22 Sep 2023 16:02:53 +0200 Subject: [PATCH 122/148] improve txpool report --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 38c5f488059..6137a9ed330 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -721,7 +721,9 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) * Transactions: {Metrics.PendingTransactionsSent,24:N0} * Hashes: {Metrics.PendingTransactionsHashesSent,24:N0} ------------------------------------------------ -Total Received: {Metrics.PendingTransactionsReceived,24:N0} +Received +* Transactions: {Metrics.PendingTransactionsReceived,24:N0} +* Hashes: {Metrics.PendingTransactionsHashesReceived,24:N0} ------------------------------------------------ Discarded at Filter Stage: 1. NotSupportedTxType {Metrics.PendingTransactionsNotSupportedTxType,24:N0} From f43c8a08250e03c1ffa6a5c927ec3fddf20de933 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 25 Sep 2023 17:08:03 +0200 Subject: [PATCH 123/148] fix naming --- .../Collections/PersistentBlobTxDistinctSortedPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 6adf53d5db5..6cffeb03977 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -67,7 +67,7 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Transaction? fullBlobTx) { - if (base.ContainsValue(hash)) + if (base.ContainsKey(hash)) { if (_blobTxCache.TryGet(hash, out fullBlobTx)) { diff --git a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs index 4420b836e55..578b29700b4 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/SortedPool.cs @@ -261,7 +261,7 @@ public IEnumerable TakeWhile(TGroupKey groupKey, Predicate where /// Key to check presence. /// True if element is present in pool. [MethodImpl(MethodImplOptions.Synchronized)] - public bool ContainsValue(TKey key) + public bool ContainsKey(TKey key) { return _cacheMap.ContainsKey(key); } diff --git a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs index b316bd7ba13..546ff325865 100644 --- a/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs +++ b/src/Nethermind/Nethermind.TxPool/TxBroadcaster.cs @@ -315,7 +315,7 @@ public bool TryGetPersistentTx(Keccak hash, out Transaction? transaction) return false; } - public bool ContainsTx(Keccak hash) => _persistentTxs.ContainsValue(hash); + public bool ContainsTx(Keccak hash) => _persistentTxs.ContainsKey(hash); public bool AddPeer(ITxPoolPeer peer) { diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 6137a9ed330..08f5dae63c4 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -591,8 +591,8 @@ public bool RemoveTransaction(Keccak? hash) } public bool ContainsTx(Keccak hash, TxType txType) => txType == TxType.Blob - ? _blobTransactions.ContainsValue(hash) - : _transactions.ContainsValue(hash) || _broadcaster.ContainsTx(hash); + ? _blobTransactions.ContainsKey(hash) + : _transactions.ContainsKey(hash) || _broadcaster.ContainsTx(hash); public bool TryGetPendingTransaction(Keccak hash, out Transaction? transaction) { From e6d1e468aa29a2bc30722840b162eb8f266622c6 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 25 Sep 2023 17:49:13 +0200 Subject: [PATCH 124/148] make naming more relevant --- .../Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs | 2 +- .../Collections/PersistentBlobTxDistinctSortedPool.cs | 4 ++-- .../Nethermind.TxPool/Collections/TxDistinctSortedPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs index f1e3b49b719..a9c35680076 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/BlobTxDistinctSortedPool.cs @@ -22,7 +22,7 @@ public BlobTxDistinctSortedPool(int capacity, IComparer comparer, I protected override IComparer GetReplacementComparer(IComparer comparer) => comparer.GetBlobReplacementComparer(); - public override void EnsureCapacity() + public override void VerifyCapacity() { if (Count > _poolCapacity && _logger.IsWarn) _logger.Warn($"Blob pool exceeds the config size {Count}/{_poolCapacity}"); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 6cffeb03977..b026bf7ec62 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -92,9 +92,9 @@ protected override bool Remove(ValueKeccak hash, Transaction tx) return base.Remove(hash, tx); } - public override void EnsureCapacity() + public override void VerifyCapacity() { - base.EnsureCapacity(); + base.VerifyCapacity(); if (_logger.IsDebug && Count == _poolCapacity) _logger.Debug($"Blob persistent storage has reached max size of {_poolCapacity}, blob txs can be evicted now"); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs index 6647bc0858a..0a10db31551 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/TxDistinctSortedPool.cs @@ -130,7 +130,7 @@ public void UpdateGroup(Address groupKey, Account groupValue, Func _poolCapacity && _logger.IsWarn) _logger.Warn($"TxPool exceeds the config size {Count}/{_poolCapacity}"); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 08f5dae63c4..34a5abc4a01 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -501,10 +501,10 @@ private void UpdateBuckets() { lock (_locker) { - _transactions.EnsureCapacity(); + _transactions.VerifyCapacity(); _transactions.UpdatePool(_accounts, _updateBucket); - _blobTransactions.EnsureCapacity(); + _blobTransactions.VerifyCapacity(); _blobTransactions.UpdatePool(_accounts, _updateBucket); } } From 1a928e63fd417055b1fc67955514598f98413ff5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Mon, 25 Sep 2023 18:11:43 +0200 Subject: [PATCH 125/148] stopwatch --- .../Collections/PersistentBlobTxDistinctSortedPool.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index b026bf7ec62..f03843fac34 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -33,8 +33,7 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) if (_logger.IsDebug) _logger.Debug("Recreating light collection of blob transactions and cache"); int numberOfTxsInDb = 0; int numberOfBlobsInDb = 0; - Stopwatch stopwatch = new(); - stopwatch.Start(); + Stopwatch stopwatch = Stopwatch.StartNew(); foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) { if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out _)) From d2e59044237f30e99eca9c36567a975906b984cc Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 26 Sep 2023 13:09:40 +0200 Subject: [PATCH 126/148] add missing metric --- .../P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs index 265c83dba0f..49867cc5bea 100644 --- a/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs +++ b/src/Nethermind/Nethermind.Network/P2P/Subprotocols/Eth/V65/Eth65ProtocolHandler.cs @@ -147,6 +147,7 @@ internal PooledTransactionsMessage FulfillPooledTransactionsRequest( txsToSend.Add(tx); packetSizeLeft -= txSize; + TxPool.Metrics.PendingTransactionsSent++; } } From c50c44b81b4d769de8286d013ab1aa1e7c590ffe Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 26 Sep 2023 17:52:25 +0200 Subject: [PATCH 127/148] move to ColumnsDb --- .../Nethermind.Db/BlobTxsColumns.cs | 10 +++ src/Nethermind/Nethermind.Db/IDbProvider.cs | 2 +- .../Nethermind.Db/StandardDbInitializer.cs | 2 +- .../Nethermind.Serialization.Rlp/RlpStream.cs | 2 +- .../Nethermind.TxPool/BlobTxStorage.cs | 38 ++++++++--- .../PersistentBlobTxDistinctSortedPool.cs | 7 +- .../Nethermind.TxPool/ITxStorage.cs | 2 +- .../Nethermind.TxPool/LightTransaction.cs | 33 ++++++++- .../Nethermind.TxPool/LightTxDecoder.cs | 67 +++++++++++++++++++ .../Nethermind.TxPool/NullBlobTxStorage.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 10 +-- 11 files changed, 152 insertions(+), 23 deletions(-) create mode 100644 src/Nethermind/Nethermind.Db/BlobTxsColumns.cs create mode 100644 src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs diff --git a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs new file mode 100644 index 00000000000..e313124ec67 --- /dev/null +++ b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs @@ -0,0 +1,10 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +namespace Nethermind.Db; + +public enum BlobTxsColumns +{ + FullBlobTxs, + LightBlobTxs +} diff --git a/src/Nethermind/Nethermind.Db/IDbProvider.cs b/src/Nethermind/Nethermind.Db/IDbProvider.cs index d6a3855a886..63a4a91ad93 100644 --- a/src/Nethermind/Nethermind.Db/IDbProvider.cs +++ b/src/Nethermind/Nethermind.Db/IDbProvider.cs @@ -32,7 +32,7 @@ public interface IDbProvider : IDisposable public IDb MetadataDb => GetDb(DbNames.Metadata); - public IDb BlobTransactionsDb => GetDb(DbNames.BlobTransactions); + public IColumnsDb BlobTransactionsDb => GetDb>(DbNames.BlobTransactions); T GetDb(string dbName) where T : class, IDb; diff --git a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs index 257d0cab5a9..e70e9bbef19 100644 --- a/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs +++ b/src/Nethermind/Nethermind.Db/StandardDbInitializer.cs @@ -64,7 +64,7 @@ private void RegisterAll(bool useReceiptsDb, bool useBlobsDb) RegisterDb(BuildRocksDbSettings(DbNames.Metadata, () => Metrics.MetadataDbReads++, () => Metrics.MetadataDbWrites++)); if (useBlobsDb) { - RegisterDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); + RegisterColumnsDb(BuildRocksDbSettings(DbNames.BlobTransactions, () => Metrics.BlobTransactionsDbReads++, () => Metrics.BlobTransactionsDbWrites++)); } } diff --git a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs index 640a17aaf84..bcef1d4ba97 100644 --- a/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs +++ b/src/Nethermind/Nethermind.Serialization.Rlp/RlpStream.cs @@ -1409,7 +1409,7 @@ public override string ToString() return $"[{nameof(RlpStream)}|{Position}/{Length}]"; } - internal byte[][] DecodeByteArrays() + public byte[][] DecodeByteArrays() { int length = ReadSequenceLength(); if (length is 0) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index abdd7cf6373..9b51d129734 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -14,23 +14,30 @@ namespace Nethermind.TxPool; public class BlobTxStorage : ITxStorage { - private readonly IDb _database; + private readonly IColumnsDb _database; + private readonly IDb _fullBlobTxsDb; + private readonly IDb _lightBlobTxsDb; private static readonly TxDecoder _txDecoder = new(); + private static readonly LightTxDecoder _lightTxDecoder = new(); public BlobTxStorage() { - _database = new MemDb(); + _database = new MemDbFactory().CreateColumnsDb("BlobTxs"); + _fullBlobTxsDb = new MemDb(); + _lightBlobTxsDb = new MemDb(); } - public BlobTxStorage(IDb database) + public BlobTxStorage(IColumnsDb database) { _database = database ?? throw new ArgumentNullException(nameof(database)); + _fullBlobTxsDb = database.GetColumnDb(BlobTxsColumns.FullBlobTxs); + _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); } public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) - => TryDecode(_database.Get(hash.Bytes), out transaction); + => TryDecodeFullTx(_fullBlobTxsDb.Get(hash.Bytes), out transaction); - private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) + private static bool TryDecodeFullTx(byte[]? txBytes, out Transaction? transaction) { if (txBytes is not null) { @@ -47,11 +54,23 @@ private static bool TryDecode(byte[]? txBytes, out Transaction? transaction) return false; } - public IEnumerable GetAll() + private static bool TryDecodeLightTx(byte[]? txBytes, out LightTransaction? lightTx) { - foreach (byte[] txBytes in _database.GetAllValues()) + if (txBytes is not null) + { + lightTx = _lightTxDecoder.Decode(txBytes); + return true; + } + + lightTx = default; + return false; + } + + public IEnumerable GetAll() + { + foreach (byte[] txBytes in _lightBlobTxsDb.GetAllValues()) { - if (TryDecode(txBytes, out Transaction? transaction)) + if (TryDecodeLightTx(txBytes, out LightTransaction? transaction)) { yield return transaction!; } @@ -74,7 +93,8 @@ public void Add(Transaction transaction) rlpStream.Encode(transaction.Timestamp); rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); - _database.Set(transaction.Hash, rlpStream.Data!); + _fullBlobTxsDb.Set(transaction.Hash, rlpStream.Data!); + _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); } public void Delete(ValueKeccak hash) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index f03843fac34..5437a764105 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -34,13 +34,12 @@ private void RecreateLightTxCollectionAndCache(ITxStorage blobTxStorage) int numberOfTxsInDb = 0; int numberOfBlobsInDb = 0; Stopwatch stopwatch = Stopwatch.StartNew(); - foreach (Transaction fullBlobTx in blobTxStorage.GetAll()) + foreach (LightTransaction lightBlobTx in blobTxStorage.GetAll()) { - if (base.TryInsert(fullBlobTx.Hash, new LightTransaction(fullBlobTx), out _)) + if (base.TryInsert(lightBlobTx.Hash, lightBlobTx, out _)) { - _blobTxCache.Set(fullBlobTx.Hash, fullBlobTx); numberOfTxsInDb++; - numberOfBlobsInDb += fullBlobTx.BlobVersionedHashes?.Length ?? 0; + numberOfBlobsInDb += lightBlobTx.BlobVersionedHashes?.Length ?? 0; } } diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 380dfe53245..8a6b06b8936 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -11,7 +11,7 @@ namespace Nethermind.TxPool; public interface ITxStorage { bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction); - IEnumerable GetAll(); + IEnumerable GetAll(); void Add(Transaction transaction); void Delete(ValueKeccak hash); } diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 76593e45626..98d55ee4899 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -3,6 +3,8 @@ using Nethermind.Core; +using Nethermind.Core.Crypto; +using Nethermind.Int256; namespace Nethermind.TxPool; @@ -13,7 +15,7 @@ public class LightTransaction : Transaction { public LightTransaction(Transaction fullTx) { - Type = TxType.Blob; + Type = fullTx.Type; Hash = fullTx.Hash; SenderAddress = fullTx.SenderAddress; Nonce = fullTx.Nonce; @@ -28,4 +30,33 @@ public LightTransaction(Transaction fullTx) PoolIndex = fullTx.PoolIndex; _size = fullTx.GetLength(); } + + public LightTransaction(TxType type, + Keccak hash, + Address sender, + UInt256 nonce, + UInt256 value, + long gasLimit, + UInt256 gasPrice, + UInt256 maxFeePerGas, + UInt256 maxFeePerBlobGas, + byte[][] blobVersionHashes, + UInt256 timestamp, + ulong poolIndex, + int size) + { + Type = type; + Hash = hash; + SenderAddress = sender; + Nonce = nonce; + Value = value; + GasLimit = gasLimit; + GasPrice = gasPrice; // means MaxPriorityFeePerGas + DecodedMaxFeePerGas = maxFeePerGas; + MaxFeePerBlobGas = maxFeePerBlobGas; + BlobVersionedHashes = blobVersionHashes; + Timestamp = timestamp; + PoolIndex = poolIndex; + _size = size; + } } diff --git a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs new file mode 100644 index 00000000000..1066c416434 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs @@ -0,0 +1,67 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using Nethermind.Core; +using Nethermind.TxPool; + +namespace Nethermind.Serialization.Rlp; + +public class LightTxDecoder : TxDecoder +{ + private int GetLength(Transaction tx) + { + return Rlp.LengthOf((byte)tx.Type) + + Rlp.LengthOf(tx.Hash) + + Rlp.LengthOf(tx.SenderAddress) + + Rlp.LengthOf(tx.Nonce) + + Rlp.LengthOf(tx.Value) + + Rlp.LengthOf(tx.GasLimit) + + Rlp.LengthOf(tx.GasPrice) + + Rlp.LengthOf(tx.DecodedMaxFeePerGas) + + Rlp.LengthOf(tx.MaxFeePerBlobGas!.Value) + + Rlp.LengthOf(tx.BlobVersionedHashes!) + + Rlp.LengthOf(tx.Timestamp) + + Rlp.LengthOf(tx.PoolIndex) + + Rlp.LengthOf(tx.GetLength()); + + } + + public byte[] Encode(Transaction tx) + { + RlpStream rlpStream = new(GetLength(tx)); + + rlpStream.Encode((byte)tx.Type); + rlpStream.Encode(tx.Hash); + rlpStream.Encode(tx.SenderAddress); + rlpStream.Encode(tx.Nonce); + rlpStream.Encode(tx.Value); + rlpStream.Encode(tx.GasLimit); + rlpStream.Encode(tx.GasPrice); + rlpStream.Encode(tx.DecodedMaxFeePerGas); + rlpStream.Encode(tx.MaxFeePerBlobGas!.Value); + rlpStream.Encode(tx.BlobVersionedHashes!); + rlpStream.Encode(tx.Timestamp); + rlpStream.Encode(tx.PoolIndex); + rlpStream.Encode(tx.GetLength()); + + return rlpStream.Data!; + } + + public LightTransaction Decode(byte[] data) + { + RlpStream rlpStream = new(data); + return new LightTransaction((TxType)rlpStream.DecodeByte(), + rlpStream.DecodeKeccak()!, + rlpStream.DecodeAddress()!, + rlpStream.DecodeUInt256(), + rlpStream.DecodeUInt256(), + rlpStream.DecodeLong(), + rlpStream.DecodeUInt256(), + rlpStream.DecodeUInt256(), + rlpStream.DecodeUInt256(), + rlpStream.DecodeByteArrays(), + rlpStream.DecodeUInt256(), + rlpStream.DecodeUlong(), + rlpStream.DecodeInt()); + } +} diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 99c307513c9..294c04ca7b1 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -19,7 +19,7 @@ public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transa return false; } - public IEnumerable GetAll() => Array.Empty(); + public IEnumerable GetAll() => Array.Empty(); public void Add(Transaction transaction) { } diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 34a5abc4a01..0545fe73e41 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -95,11 +95,16 @@ public TxPool(IEthereumEcdsa ecdsa, MemoryAllowance.MemPoolSize = txPoolConfig.Size; AddNodeInfoEntryForTxPool(); + // Capture closures once rather than per invocation + _updateBucket = UpdateBucket; + + _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); + _transactions = new TxDistinctSortedPool(MemoryAllowance.MemPoolSize, comparer, logManager); _blobTransactions = txPoolConfig is { BlobSupportEnabled: true, PersistentBlobStorageEnabled: true } ? new PersistentBlobTxDistinctSortedPool(blobTxStorage, _txPoolConfig, comparer, logManager) : new BlobTxDistinctSortedPool(txPoolConfig.BlobSupportEnabled ? _txPoolConfig.InMemoryBlobPoolSize : 0, comparer, logManager); - _broadcaster = new TxBroadcaster(comparer, TimerFactory.Default, txPoolConfig, chainHeadInfoProvider, logManager, transactionsGossipPolicy); + if (_blobTransactions.Count > 0) _blobTransactions.UpdatePool(_accounts, _updateBucket); _headInfo.HeadChanged += OnHeadChange; @@ -133,9 +138,6 @@ public TxPool(IEthereumEcdsa ecdsa, _postHashFilters = postHashFilters.ToArray(); - // Capture closures once rather than per invocation - _updateBucket = UpdateBucket; - int? reportMinutes = txPoolConfig.ReportMinutes; if (_logger.IsInfo && reportMinutes.HasValue) { From 16d99b3e701133d7c45272da53861ef84689f127 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 28 Sep 2023 13:22:50 +0200 Subject: [PATCH 128/148] optimize full blobs db by prefixing key with timestamp --- .../Nethermind.TxPool/BlobTxStorage.cs | 99 ++++++++++++------- .../Nethermind.TxPool/LightTransaction.cs | 10 +- .../Nethermind.TxPool/LightTxDecoder.cs | 23 +++-- 3 files changed, 83 insertions(+), 49 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 9b51d129734..e9ca089308f 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using System; +using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Nethermind.Core; @@ -14,7 +15,6 @@ namespace Nethermind.TxPool; public class BlobTxStorage : ITxStorage { - private readonly IColumnsDb _database; private readonly IDb _fullBlobTxsDb; private readonly IDb _lightBlobTxsDb; private static readonly TxDecoder _txDecoder = new(); @@ -22,31 +22,72 @@ public class BlobTxStorage : ITxStorage public BlobTxStorage() { - _database = new MemDbFactory().CreateColumnsDb("BlobTxs"); _fullBlobTxsDb = new MemDb(); _lightBlobTxsDb = new MemDb(); } public BlobTxStorage(IColumnsDb database) { - _database = database ?? throw new ArgumentNullException(nameof(database)); _fullBlobTxsDb = database.GetColumnDb(BlobTxsColumns.FullBlobTxs); _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); } public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) - => TryDecodeFullTx(_fullBlobTxsDb.Get(hash.Bytes), out transaction); + { + DecodeTimestamp(_lightBlobTxsDb.Get(hash.Bytes), out UInt256 timestamp); + Span txHashPrefixed = stackalloc byte[64]; + GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); + + byte[]? txBytes = _fullBlobTxsDb.Get(txHashPrefixed); + return TryDecodeFullTx(txBytes, out transaction); + } + + public IEnumerable GetAll() + { + foreach (byte[] txBytes in _lightBlobTxsDb.GetAllValues()) + { + if (TryDecodeLightTx(txBytes, out LightTransaction? transaction)) + { + yield return transaction!; + } + } + } + + public void Add(Transaction transaction) + { + if (transaction?.Hash is null) + { + throw new ArgumentNullException(nameof(transaction)); + } + + Span txHashPrefixed = stackalloc byte[64]; + GetHashPrefixedByTimestamp(transaction.Timestamp, transaction.Hash, txHashPrefixed); + + int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); + RlpStream rlpStream = new(length); + rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); + + _fullBlobTxsDb.Set(txHashPrefixed, rlpStream.Data!); + _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); + } + + public void Delete(ValueKeccak hash) + { + DecodeTimestamp(_lightBlobTxsDb.Get(hash.Bytes), out UInt256 timestamp); + + Span txHashPrefixed = stackalloc byte[64]; + GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); + + _fullBlobTxsDb.Remove(txHashPrefixed); + _lightBlobTxsDb.Remove(hash.BytesAsSpan); + } private static bool TryDecodeFullTx(byte[]? txBytes, out Transaction? transaction) { if (txBytes is not null) { RlpStream rlpStream = new(txBytes); - Address sender = rlpStream.DecodeAddress()!; - UInt256 timestamp = rlpStream.DecodeUInt256(); transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); - transaction.SenderAddress = sender; - transaction.Timestamp = timestamp; return true; } @@ -66,37 +107,25 @@ private static bool TryDecodeLightTx(byte[]? txBytes, out LightTransaction? ligh return false; } - public IEnumerable GetAll() + private static void DecodeTimestamp(byte[]? txBytes, out UInt256 timestamp) { - foreach (byte[] txBytes in _lightBlobTxsDb.GetAllValues()) - { - if (TryDecodeLightTx(txBytes, out LightTransaction? transaction)) - { - yield return transaction!; - } - } + timestamp = txBytes is not null ? _lightTxDecoder.DecodeTimestamp(txBytes) : default; } - public void Add(Transaction transaction) + private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueKeccak hash, Span txHashPrefixed) { - if (transaction?.Hash is null) - { - throw new ArgumentNullException(nameof(transaction)); - } - - int length = Rlp.LengthOf(transaction.SenderAddress); - length += Rlp.LengthOf(transaction.Timestamp); - length += _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); - - RlpStream rlpStream = new(length); - rlpStream.Encode(transaction.SenderAddress); - rlpStream.Encode(transaction.Timestamp); - rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); - - _fullBlobTxsDb.Set(transaction.Hash, rlpStream.Data!); - _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); + timestamp.WriteBigEndian(txHashPrefixed); + hash.Bytes.CopyTo(txHashPrefixed[32..]); } +} - public void Delete(ValueKeccak hash) - => _database.Remove(hash.Bytes); +internal static class UInt256Extensions +{ + public static void WriteBigEndian(in this UInt256 value, Span output) + { + BinaryPrimitives.WriteUInt64BigEndian(output.Slice(0, 8), value.u3); + BinaryPrimitives.WriteUInt64BigEndian(output.Slice(8, 8), value.u2); + BinaryPrimitives.WriteUInt64BigEndian(output.Slice(16, 8), value.u1); + BinaryPrimitives.WriteUInt64BigEndian(output.Slice(24, 8), value.u0); + } } diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 98d55ee4899..1c9cb8d0418 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -15,7 +15,7 @@ public class LightTransaction : Transaction { public LightTransaction(Transaction fullTx) { - Type = fullTx.Type; + Type = TxType.Blob; Hash = fullTx.Hash; SenderAddress = fullTx.SenderAddress; Nonce = fullTx.Nonce; @@ -31,21 +31,21 @@ public LightTransaction(Transaction fullTx) _size = fullTx.GetLength(); } - public LightTransaction(TxType type, - Keccak hash, + public LightTransaction( + UInt256 timestamp, Address sender, UInt256 nonce, + Keccak hash, UInt256 value, long gasLimit, UInt256 gasPrice, UInt256 maxFeePerGas, UInt256 maxFeePerBlobGas, byte[][] blobVersionHashes, - UInt256 timestamp, ulong poolIndex, int size) { - Type = type; + Type = TxType.Blob; Hash = hash; SenderAddress = sender; Nonce = nonce; diff --git a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs index 1066c416434..93fb494a402 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs @@ -2,6 +2,7 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; +using Nethermind.Int256; using Nethermind.TxPool; namespace Nethermind.Serialization.Rlp; @@ -10,17 +11,16 @@ public class LightTxDecoder : TxDecoder { private int GetLength(Transaction tx) { - return Rlp.LengthOf((byte)tx.Type) - + Rlp.LengthOf(tx.Hash) + return Rlp.LengthOf(tx.Timestamp) + Rlp.LengthOf(tx.SenderAddress) + Rlp.LengthOf(tx.Nonce) + + Rlp.LengthOf(tx.Hash) + Rlp.LengthOf(tx.Value) + Rlp.LengthOf(tx.GasLimit) + Rlp.LengthOf(tx.GasPrice) + Rlp.LengthOf(tx.DecodedMaxFeePerGas) + Rlp.LengthOf(tx.MaxFeePerBlobGas!.Value) + Rlp.LengthOf(tx.BlobVersionedHashes!) - + Rlp.LengthOf(tx.Timestamp) + Rlp.LengthOf(tx.PoolIndex) + Rlp.LengthOf(tx.GetLength()); @@ -30,17 +30,16 @@ public byte[] Encode(Transaction tx) { RlpStream rlpStream = new(GetLength(tx)); - rlpStream.Encode((byte)tx.Type); - rlpStream.Encode(tx.Hash); + rlpStream.Encode(tx.Timestamp); rlpStream.Encode(tx.SenderAddress); rlpStream.Encode(tx.Nonce); + rlpStream.Encode(tx.Hash); rlpStream.Encode(tx.Value); rlpStream.Encode(tx.GasLimit); rlpStream.Encode(tx.GasPrice); rlpStream.Encode(tx.DecodedMaxFeePerGas); rlpStream.Encode(tx.MaxFeePerBlobGas!.Value); rlpStream.Encode(tx.BlobVersionedHashes!); - rlpStream.Encode(tx.Timestamp); rlpStream.Encode(tx.PoolIndex); rlpStream.Encode(tx.GetLength()); @@ -50,18 +49,24 @@ public byte[] Encode(Transaction tx) public LightTransaction Decode(byte[] data) { RlpStream rlpStream = new(data); - return new LightTransaction((TxType)rlpStream.DecodeByte(), - rlpStream.DecodeKeccak()!, + return new LightTransaction( + rlpStream.DecodeUInt256(), rlpStream.DecodeAddress()!, rlpStream.DecodeUInt256(), + rlpStream.DecodeKeccak()!, rlpStream.DecodeUInt256(), rlpStream.DecodeLong(), rlpStream.DecodeUInt256(), rlpStream.DecodeUInt256(), rlpStream.DecodeUInt256(), rlpStream.DecodeByteArrays(), - rlpStream.DecodeUInt256(), rlpStream.DecodeUlong(), rlpStream.DecodeInt()); } + + public UInt256 DecodeTimestamp(byte[] data) + { + RlpStream rlpStream = new(data); + return rlpStream.DecodeUInt256(); + } } From a19c4cebcc93c88bf3cb675fa714f6ffb285305d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Fri, 29 Sep 2023 16:30:21 +0200 Subject: [PATCH 129/148] fix files encoding --- src/Nethermind/Nethermind.Db/BlobTxsColumns.cs | 2 +- src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs index e313124ec67..56b4f1d4848 100644 --- a/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs +++ b/src/Nethermind/Nethermind.Db/BlobTxsColumns.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only namespace Nethermind.Db; diff --git a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs index 93fb494a402..6b8d0b4ce76 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; From a804f515eeb64f7a124ca54d9b6d014a5fe3b660 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 3 Oct 2023 16:53:12 +0200 Subject: [PATCH 130/148] fix and improve blob storage --- .../Nethermind.TxPool/BlobTxStorage.cs | 17 +++++------------ .../PersistentBlobTxDistinctSortedPool.cs | 6 +++--- src/Nethermind/Nethermind.TxPool/ITxStorage.cs | 5 +++-- .../Nethermind.TxPool/LightTxDecoder.cs | 11 ++--------- .../Nethermind.TxPool/NullBlobTxStorage.cs | 5 +++-- 5 files changed, 16 insertions(+), 28 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index e9ca089308f..03fd6f267bf 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -32,14 +32,13 @@ public BlobTxStorage(IColumnsDb database) _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); } - public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) + public bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) { - DecodeTimestamp(_lightBlobTxsDb.Get(hash.Bytes), out UInt256 timestamp); Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); byte[]? txBytes = _fullBlobTxsDb.Get(txHashPrefixed); - return TryDecodeFullTx(txBytes, out transaction); + return TryDecodeFullTx(txBytes, sender, out transaction); } public IEnumerable GetAll() @@ -71,10 +70,8 @@ public void Add(Transaction transaction) _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); } - public void Delete(ValueKeccak hash) + public void Delete(ValueKeccak hash, UInt256 timestamp) { - DecodeTimestamp(_lightBlobTxsDb.Get(hash.Bytes), out UInt256 timestamp); - Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); @@ -82,12 +79,13 @@ public void Delete(ValueKeccak hash) _lightBlobTxsDb.Remove(hash.BytesAsSpan); } - private static bool TryDecodeFullTx(byte[]? txBytes, out Transaction? transaction) + private static bool TryDecodeFullTx(byte[]? txBytes, Address sender, out Transaction? transaction) { if (txBytes is not null) { RlpStream rlpStream = new(txBytes); transaction = Rlp.Decode(rlpStream, RlpBehaviors.InMempoolForm); + transaction.SenderAddress = sender; return true; } @@ -107,11 +105,6 @@ private static bool TryDecodeLightTx(byte[]? txBytes, out LightTransaction? ligh return false; } - private static void DecodeTimestamp(byte[]? txBytes, out UInt256 timestamp) - { - timestamp = txBytes is not null ? _lightTxDecoder.DecodeTimestamp(txBytes) : default; - } - private void GetHashPrefixedByTimestamp(UInt256 timestamp, ValueKeccak hash, Span txHashPrefixed) { timestamp.WriteBigEndian(txHashPrefixed); diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 5437a764105..452000450ff 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -65,14 +65,14 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Transaction? fullBlobTx) { - if (base.ContainsKey(hash)) + if (base.TryGetValue(hash, out Transaction? lightTx)) { if (_blobTxCache.TryGet(hash, out fullBlobTx)) { return true; } - if (_blobTxStorage.TryGet(hash, out fullBlobTx)) + if (_blobTxStorage.TryGet(hash, lightTx.SenderAddress!, lightTx.Timestamp, out fullBlobTx)) { _blobTxCache.Set(hash, fullBlobTx); return true; @@ -86,7 +86,7 @@ public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Trans protected override bool Remove(ValueKeccak hash, Transaction tx) { _blobTxCache.Delete(hash); - _blobTxStorage.Delete(hash); + _blobTxStorage.Delete(hash, tx.Timestamp); return base.Remove(hash, tx); } diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 8a6b06b8936..445de45ff75 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -5,13 +5,14 @@ using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Int256; namespace Nethermind.TxPool; public interface ITxStorage { - bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction); + bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction); IEnumerable GetAll(); void Add(Transaction transaction); - void Delete(ValueKeccak hash); + void Delete(ValueKeccak hash, UInt256 timestamp); } diff --git a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs index 6b8d0b4ce76..aa5680d7427 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTxDecoder.cs @@ -2,10 +2,9 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; -using Nethermind.Int256; -using Nethermind.TxPool; +using Nethermind.Serialization.Rlp; -namespace Nethermind.Serialization.Rlp; +namespace Nethermind.TxPool; public class LightTxDecoder : TxDecoder { @@ -63,10 +62,4 @@ public LightTransaction Decode(byte[] data) rlpStream.DecodeUlong(), rlpStream.DecodeInt()); } - - public UInt256 DecodeTimestamp(byte[] data) - { - RlpStream rlpStream = new(data); - return rlpStream.DecodeUInt256(); - } } diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 294c04ca7b1..0969732e816 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -6,6 +6,7 @@ using System.Diagnostics.CodeAnalysis; using Nethermind.Core; using Nethermind.Core.Crypto; +using Nethermind.Int256; namespace Nethermind.TxPool; @@ -13,7 +14,7 @@ public class NullBlobTxStorage : ITxStorage { public static NullBlobTxStorage Instance { get; } = new(); - public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transaction) + public bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) { transaction = default; return false; @@ -23,5 +24,5 @@ public bool TryGet(ValueKeccak hash, [NotNullWhen(true)] out Transaction? transa public void Add(Transaction transaction) { } - public void Delete(ValueKeccak hash) { } + public void Delete(ValueKeccak hash, UInt256 timestamp) { } } From f9e703089133adbdd22b72a55893d4e62a88076d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 3 Oct 2023 17:01:59 +0200 Subject: [PATCH 131/148] fix tests --- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index be6f1118990..70be9a1bf1e 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -164,7 +164,7 @@ public void should_add_blob_tx_and_return_when_requested([Values(true, false)] b blobTxReturned.Should().BeEquivalentTo(blobTxAdded); - blobTxStorage.TryGet(blobTxAdded.Hash, out Transaction blobTxFromDb).Should().Be(isPersistentStorage); // additional check for persistent db + blobTxStorage.TryGet(blobTxAdded.Hash, blobTxAdded.SenderAddress!, blobTxAdded.Timestamp, out Transaction blobTxFromDb).Should().Be(isPersistentStorage); // additional check for persistent db if (isPersistentStorage) { blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options @@ -184,7 +184,7 @@ public void should_not_throw_when_asking_for_non_existing_tx() _txPool.TryGetPendingTransaction(TestItem.KeccakA, out Transaction blobTxReturned).Should().BeFalse(); blobTxReturned.Should().BeNull(); - blobTxStorage.TryGet(TestItem.KeccakA, out Transaction blobTxFromDb).Should().BeFalse(); + blobTxStorage.TryGet(TestItem.KeccakA, TestItem.AddressA, UInt256.One, out Transaction blobTxFromDb).Should().BeFalse(); blobTxFromDb.Should().BeNull(); } @@ -290,7 +290,7 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() _txPool.GetPendingBlobTransactionsCount().Should().Be(1); _txPool.TryGetPendingTransaction(oldTx.Hash!, out Transaction blobTxReturned).Should().BeTrue(); blobTxReturned.Should().BeEquivalentTo(oldTx); - blobTxStorage.TryGet(oldTx.Hash, out Transaction blobTxFromDb).Should().BeTrue(); + blobTxStorage.TryGet(oldTx.Hash, oldTx.SenderAddress!, oldTx.Timestamp, out Transaction blobTxFromDb).Should().BeTrue(); blobTxFromDb.Should().BeEquivalentTo(oldTx, options => options .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... @@ -300,8 +300,8 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() _txPool.GetPendingBlobTransactionsCount().Should().Be(1); _txPool.TryGetPendingTransaction(newTx.Hash!, out blobTxReturned).Should().BeTrue(); blobTxReturned.Should().BeEquivalentTo(newTx); - blobTxStorage.TryGet(oldTx.Hash, out blobTxFromDb).Should().BeFalse(); - blobTxStorage.TryGet(newTx.Hash, out blobTxFromDb).Should().BeTrue(); + blobTxStorage.TryGet(oldTx.Hash, oldTx.SenderAddress, oldTx.Timestamp, out blobTxFromDb).Should().BeFalse(); + blobTxStorage.TryGet(newTx.Hash, newTx.SenderAddress!, newTx.Timestamp, out blobTxFromDb).Should().BeTrue(); blobTxFromDb.Should().BeEquivalentTo(newTx, options => options .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... From ca5a37ee5386d9179c6ba5300ac4f300769030a9 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 4 Oct 2023 10:34:35 +0200 Subject: [PATCH 132/148] adjust tests to require sender of full blob txs loaded from db --- .../Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index 70be9a1bf1e..d1a3bfa140c 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -168,9 +168,8 @@ public void should_add_blob_tx_and_return_when_requested([Values(true, false)] b if (isPersistentStorage) { blobTxFromDb.Should().BeEquivalentTo(blobTxAdded, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex } } @@ -292,9 +291,8 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() blobTxReturned.Should().BeEquivalentTo(oldTx); blobTxStorage.TryGet(oldTx.Hash, oldTx.SenderAddress!, oldTx.Timestamp, out Transaction blobTxFromDb).Should().BeTrue(); blobTxFromDb.Should().BeEquivalentTo(oldTx, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex _txPool.SubmitTx(newTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); _txPool.GetPendingBlobTransactionsCount().Should().Be(1); @@ -303,9 +301,8 @@ public void should_remove_replaced_blob_tx_from_persistent_storage_and_cache() blobTxStorage.TryGet(oldTx.Hash, oldTx.SenderAddress, oldTx.Timestamp, out blobTxFromDb).Should().BeFalse(); blobTxStorage.TryGet(newTx.Hash, newTx.SenderAddress!, newTx.Timestamp, out blobTxFromDb).Should().BeTrue(); blobTxFromDb.Should().BeEquivalentTo(newTx, options => options - .Excluding(t => t.SenderAddress) // sender is not encoded/decoded... - .Excluding(t => t.GasBottleneck) // ...as well as GasBottleneck... - .Excluding(t => t.PoolIndex)); // ...and PoolIndex + .Excluding(t => t.GasBottleneck) // GasBottleneck is not encoded/decoded... + .Excluding(t => t.PoolIndex)); // ...as well as PoolIndex } [Test] From 8e2ce3549053a087a78b9c55a6f37261c694334e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 4 Oct 2023 11:19:08 +0200 Subject: [PATCH 133/148] refactor future nonce filter --- ...erSenderFilter.cs => FutureNonceFilter.cs} | 19 ++++++++++--------- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) rename src/Nethermind/Nethermind.TxPool/Filters/{MaxPendingTxsPerSenderFilter.cs => FutureNonceFilter.cs} (53%) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/FutureNonceFilter.cs similarity index 53% rename from src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs rename to src/Nethermind/Nethermind.TxPool/Filters/FutureNonceFilter.cs index 053914b5caa..e3e74abd59b 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/MaxPendingTxsPerSenderFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/FutureNonceFilter.cs @@ -2,21 +2,17 @@ // SPDX-License-Identifier: LGPL-3.0-only using Nethermind.Core; -using Nethermind.TxPool.Collections; +using Nethermind.Int256; namespace Nethermind.TxPool.Filters; -public class MaxPendingTxsPerSenderFilter : IIncomingTxFilter +public class FutureNonceFilter : IIncomingTxFilter { private readonly ITxPoolConfig _txPoolConfig; - private readonly TxDistinctSortedPool _txs; - private readonly TxDistinctSortedPool _blobTxs; - public MaxPendingTxsPerSenderFilter(ITxPoolConfig txPoolConfig, TxDistinctSortedPool txs, TxDistinctSortedPool blobTxs) + public FutureNonceFilter(ITxPoolConfig txPoolConfig) { _txPoolConfig = txPoolConfig; - _txs = txs; - _blobTxs = blobTxs; } public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions txHandlingOptions) @@ -25,14 +21,19 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO ? _txPoolConfig.MaxPendingBlobTxsPerSender : _txPoolConfig.MaxPendingTxsPerSender); + // MaxPendingTxsPerSender/MaxPendingBlobTxsPerSender equal 0 means no limit if (relevantMaxPendingTxsPerSender == 0) { return AcceptTxResult.Accepted; } - TxDistinctSortedPool relevantTxPool = (tx.SupportsBlobs ? _blobTxs : _txs); + UInt256 currentNonce = state.SenderAccount.Nonce; + bool overflow = UInt256.AddOverflow(currentNonce, (UInt256)relevantMaxPendingTxsPerSender, out UInt256 maxAcceptedNonce); - if (relevantTxPool.GetBucketCount(tx.SenderAddress!) > relevantMaxPendingTxsPerSender) + // Overflow means that gap between current nonce of sender and UInt256.MaxValue is lower than allowed number + // of pending transactions. As lower nonces were rejected earlier, here it means tx accepted. + // So we are rejecting tx only if there is no overflow. + if (tx.Nonce > maxAcceptedNonce && !overflow) { Metrics.PendingTransactionsNonceTooFarInFuture++; return AcceptTxResult.NonceTooFarInFuture; diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 0545fe73e41..4fc92ef11c5 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -122,10 +122,10 @@ public TxPool(IEthereumEcdsa ecdsa, new AlreadyKnownTxFilter(_hashCache, _logger), new UnknownSenderFilter(ecdsa, _logger), new TxTypeTxFilter(_transactions, _blobTransactions), // has to be after UnknownSenderFilter as it uses sender - new MaxPendingTxsPerSenderFilter(txPoolConfig, _transactions, _blobTransactions), new BalanceZeroFilter(thereIsPriorityContract, _logger), new BalanceTooLowFilter(_transactions, _blobTransactions, _logger), new LowNonceFilter(_logger), // has to be after UnknownSenderFilter as it uses sender + new FutureNonceFilter(txPoolConfig), new GapNonceFilter(_transactions, _blobTransactions, _logger), }; From 1542e2e926bd03e82087d19fba0966ed38e8e73e Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 4 Oct 2023 14:07:38 +0200 Subject: [PATCH 134/148] simplify and improve block building --- .../Producers/TxPoolTxSource.cs | 23 ++++++------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 81c7f1243e1..f9383e11a24 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -175,26 +175,17 @@ private bool TryUpdateBlobGasPrice(Transaction fullBlobTx, BlockHeader parent, o private IEnumerable PickBlobTxsBetterThanCurrentTx(ArrayPoolList selectedBlobTxs, Transaction tx, IComparer comparer) { - if (selectedBlobTxs.Count > 0) + while (selectedBlobTxs.Count > 0) { - using ArrayPoolList txsToRemove = new(selectedBlobTxs.Count); - - foreach (Transaction blobTx in selectedBlobTxs) + Transaction blobTx = selectedBlobTxs[0]; + if (comparer.Compare(blobTx, tx) > 0) { - if (comparer.Compare(blobTx, tx) > 0) - { - yield return blobTx; - txsToRemove.Add(blobTx); - } - else - { - break; - } + yield return blobTx; + selectedBlobTxs.Remove(blobTx); } - - foreach (Transaction txToRemove in txsToRemove) + else { - selectedBlobTxs.Remove(txToRemove); + break; } } } From 574725d3db8bc33832d2ad22e23fc45b65259b39 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Wed, 4 Oct 2023 17:41:01 +0200 Subject: [PATCH 135/148] refactor TxPoolTxSource --- .../Producers/TxPoolTxSource.cs | 136 ++++++++++-------- 1 file changed, 73 insertions(+), 63 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index f9383e11a24..1e0091198dd 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -58,11 +58,71 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi IEnumerable blobTransactions = GetOrderedTransactions(pendingBlobTransactionsEquivalences, comparer); if (_logger.IsDebug) _logger.Debug($"Collecting pending transactions at block gas limit {gasLimit}."); + int checkedTransactions = 0; int selectedTransactions = 0; - int i = 0; + using ArrayPoolList selectedBlobTxs = new(Eip4844Constants.MaxBlobsPerBlock); + + SelectBlobTransactions(blobTransactions, parent, selectedBlobTxs); + + foreach (Transaction tx in transactions) + { + checkedTransactions++; + + if (tx.SenderAddress is null) + { + _transactionPool.RemoveTransaction(tx.Hash!); + if (_logger.IsDebug) _logger.Debug($"Rejecting (null sender) {tx.ToShortString()}"); + continue; + } + + bool success = _txFilterPipeline.Execute(tx, parent); + if (!success) continue; + + foreach (Transaction blobTx in PickBlobTxsBetterThanCurrentTx(selectedBlobTxs, tx, comparer)) + { + yield return blobTx; + } + + if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); + + selectedTransactions++; + yield return tx; + } + + if (selectedBlobTxs.Count > 0) + { + foreach (Transaction blobTx in selectedBlobTxs) + { + yield return blobTx; + } + } + + if (_logger.IsDebug) _logger.Debug($"Potentially selected {selectedTransactions} out of {checkedTransactions} pending transactions checked."); + } + + private IEnumerable PickBlobTxsBetterThanCurrentTx(ArrayPoolList selectedBlobTxs, Transaction tx, IComparer comparer) + { + while (selectedBlobTxs.Count > 0) + { + Transaction blobTx = selectedBlobTxs[0]; + if (comparer.Compare(blobTx, tx) > 0) + { + yield return blobTx; + selectedBlobTxs.Remove(blobTx); + } + else + { + break; + } + } + } + + private void SelectBlobTransactions(IEnumerable blobTransactions, BlockHeader parent, ArrayPoolList selectedBlobTxs) + { + int checkedBlobTransactions = 0; + int selectedBlobTransactions = 0; int blobsCounter = 0; UInt256 blobGasPrice = UInt256.Zero; - using ArrayPoolList selectedBlobTxs = new(Eip4844Constants.MaxBlobsPerBlock); foreach (Transaction blobTx in blobTransactions) { @@ -72,14 +132,21 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi break; } + checkedBlobTransactions++; + + int txAmountOfBlobs = blobTx.BlobVersionedHashes?.Length ?? 0; + if (blobsCounter + txAmountOfBlobs > Eip4844Constants.MaxBlobsPerBlock) + { + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, not enough blob space."); + continue; + } + if (!TryGetFullBlobTx(blobTx, out Transaction fullBlobTx)) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to get full version of this blob tx from TxPool."); continue; } - i++; - bool success = _txFilterPipeline.Execute(fullBlobTx, parent); if (!success) continue; @@ -94,54 +161,14 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi continue; } - int txAmountOfBlobs = fullBlobTx.BlobVersionedHashes?.Length ?? 0; - if (blobsCounter + txAmountOfBlobs > Eip4844Constants.MaxBlobsPerBlock) - { - if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, not enough blob space."); - continue; - } - blobsCounter += txAmountOfBlobs; if (_logger.IsTrace) _logger.Trace($"Selected shard blob tx {fullBlobTx.ToShortString()} to be potentially included in block, total blobs included: {blobsCounter}."); - selectedTransactions++; + selectedBlobTransactions++; selectedBlobTxs.Add(fullBlobTx); } - foreach (Transaction tx in transactions) - { - i++; - - if (tx.SenderAddress is null) - { - _transactionPool.RemoveTransaction(tx.Hash!); - if (_logger.IsDebug) _logger.Debug($"Rejecting (null sender) {tx.ToShortString()}"); - continue; - } - - bool success = _txFilterPipeline.Execute(tx, parent); - if (!success) continue; - - foreach (Transaction blobTx in PickBlobTxsBetterThanCurrentTx(selectedBlobTxs, tx, comparer)) - { - yield return blobTx; - } - - if (_logger.IsTrace) _logger.Trace($"Selected {tx.ToShortString()} to be potentially included in block."); - - selectedTransactions++; - yield return tx; - } - - if (selectedBlobTxs.Count > 0) - { - foreach (Transaction blobTx in selectedBlobTxs) - { - yield return blobTx; - } - } - - if (_logger.IsDebug) _logger.Debug($"Potentially selected {selectedTransactions} out of {i} pending transactions checked."); + if (_logger.IsDebug) _logger.Debug($"Potentially selected {selectedBlobTransactions} out of {checkedBlobTransactions} pending blob transactions checked."); } private bool TryGetFullBlobTx(Transaction blobTx, [NotNullWhen(true)] out Transaction? fullBlobTx) @@ -173,23 +200,6 @@ private bool TryUpdateBlobGasPrice(Transaction fullBlobTx, BlockHeader parent, o return true; } - private IEnumerable PickBlobTxsBetterThanCurrentTx(ArrayPoolList selectedBlobTxs, Transaction tx, IComparer comparer) - { - while (selectedBlobTxs.Count > 0) - { - Transaction blobTx = selectedBlobTxs[0]; - if (comparer.Compare(blobTx, tx) > 0) - { - yield return blobTx; - selectedBlobTxs.Remove(blobTx); - } - else - { - break; - } - } - } - protected virtual IEnumerable GetOrderedTransactions(IDictionary pendingTransactions, IComparer comparer) => Order(pendingTransactions, comparer); From 91d420c6c7bc8750cdccc8e5e347ea8ce535d190 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Thu, 5 Oct 2023 13:55:14 +0200 Subject: [PATCH 136/148] fix block building - move loading full tx from db after all filters --- .../Producers/TxPoolTxSource.cs | 21 ++++++++++--------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 1e0091198dd..186889eb521 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -141,23 +141,24 @@ private void SelectBlobTransactions(IEnumerable blobTransactions, B continue; } - if (!TryGetFullBlobTx(blobTx, out Transaction fullBlobTx)) + if (blobGasPrice.IsZero && !TryUpdateBlobGasPrice(blobTx, parent, out blobGasPrice)) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to get full version of this blob tx from TxPool."); continue; } - bool success = _txFilterPipeline.Execute(fullBlobTx, parent); - if (!success) continue; - - if (blobGasPrice.IsZero && !TryUpdateBlobGasPrice(fullBlobTx, parent, out blobGasPrice)) + if (blobGasPrice > blobTx.MaxFeePerBlobGas) { + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, data gas fee is too low."); continue; } - if (blobGasPrice > fullBlobTx.MaxFeePerBlobGas) + bool success = _txFilterPipeline.Execute(blobTx, parent); + if (!success) continue; + + if (!TryGetFullBlobTx(blobTx, out Transaction fullBlobTx)) { - if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, data gas fee is too low."); + if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to get full version of this blob tx from TxPool."); continue; } @@ -182,18 +183,18 @@ private bool TryGetFullBlobTx(Transaction blobTx, [NotNullWhen(true)] out Transa return blobTx.Hash is not null && _transactionPool.TryGetPendingBlobTransaction(blobTx.Hash, out fullBlobTx); } - private bool TryUpdateBlobGasPrice(Transaction fullBlobTx, BlockHeader parent, out UInt256 blobGasPrice) + private bool TryUpdateBlobGasPrice(Transaction lightBlobTx, BlockHeader parent, out UInt256 blobGasPrice) { ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, _specProvider.GetSpec(parent)); if (excessDataGas is null) { - if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); + if (_logger.IsTrace) _logger.Trace($"Declining {lightBlobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); blobGasPrice = UInt256.Zero; return false; } if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(excessDataGas.Value, out blobGasPrice)) { - if (_logger.IsTrace) _logger.Trace($"Declining {fullBlobTx.ToShortString()}, failed to calculate data gas price."); + if (_logger.IsTrace) _logger.Trace($"Declining {lightBlobTx.ToShortString()}, failed to calculate data gas price."); blobGasPrice = UInt256.Zero; return false; } From 76b6e2527bae588d016ae4e222054e39df76bd94 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:08:10 +0200 Subject: [PATCH 137/148] refactor low fee filtering --- .../Filters/FeeTooLowFilter.cs | 10 +----- .../Filters/PriorityFeeTooLowFilter.cs | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 9 deletions(-) create mode 100644 src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs diff --git a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs index 49afaa854e2..432930567b1 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/FeeTooLowFilter.cs @@ -3,7 +3,6 @@ using System.Diagnostics; using Nethermind.Core; -using Nethermind.Core.Extensions; using Nethermind.Core.Specs; using Nethermind.Int256; using Nethermind.Logging; @@ -12,7 +11,7 @@ namespace Nethermind.TxPool.Filters { /// - /// Filters out transactions where gas fee properties were set too low or where the sender has not enough balance. + /// Filters out transactions where gas fee properties were set too low. /// internal sealed class FeeTooLowFilter : IIncomingTxFilter { @@ -35,13 +34,6 @@ public FeeTooLowFilter(IChainHeadInfoProvider headInfo, TxDistinctSortedPool txs public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { - if (tx.SupportsBlobs && tx.MaxPriorityFeePerGas < 1.GWei()) - { - Metrics.PendingTransactionsTooLowFee++; - if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, too low payable gas price with options {handlingOptions} from {new StackTrace()}"); - return AcceptTxResult.FeeTooLow.WithMessage($"MaxPriorityFeePerGas for blob transaction needs to be at least {1.GWei()} (1 GWei), is {tx.MaxPriorityFeePerGas}."); - } - bool isLocal = (handlingOptions & TxHandlingOptions.PersistentBroadcast) != 0; if (isLocal) { diff --git a/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs new file mode 100644 index 00000000000..3b455788ef7 --- /dev/null +++ b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs @@ -0,0 +1,31 @@ +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-License-Identifier: LGPL-3.0-only + +using System.Diagnostics; +using Nethermind.Core; +using Nethermind.Logging; + +namespace Nethermind.TxPool.Filters; + +public class PriorityFeeTooLowFilter : IIncomingTxFilter +{ + private readonly ILogger _logger; + private const int OneGWei = 1_000_000_000; + + public PriorityFeeTooLowFilter(ILogger logger) + { + _logger = logger; + } + + public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) + { + if (tx.SupportsBlobs && tx.MaxPriorityFeePerGas < OneGWei) + { + Metrics.PendingTransactionsTooLowPriorityFee++; + if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, too low priority fee with options {handlingOptions} from {new StackTrace()}"); + return AcceptTxResult.FeeTooLow.WithMessage($"MaxPriorityFeePerGas for blob transaction needs to be at least {OneGWei} (1 GWei), is {tx.MaxPriorityFeePerGas}."); + } + + return AcceptTxResult.Accepted; + } +} From 3b150cdd71d3745c1eb6e1ce02457eff4c16aeb5 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:08:31 +0200 Subject: [PATCH 138/148] add new metric --- src/Nethermind/Nethermind.TxPool/Metrics.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Metrics.cs b/src/Nethermind/Nethermind.TxPool/Metrics.cs index 1668bd08099..53721d957fe 100644 --- a/src/Nethermind/Nethermind.TxPool/Metrics.cs +++ b/src/Nethermind/Nethermind.TxPool/Metrics.cs @@ -38,8 +38,11 @@ public static class Metrics public static long PendingTransactionsNonceGap { get; set; } [CounterMetric] - [Description( - "Number of pending transactions received that were ignored because of fee lower than the lowest fee in transaction pool.")] + [Description("Number of pending transactions received that were ignored because of priority fee lower than minimal requirement.")] + public static long PendingTransactionsTooLowPriorityFee { get; set; } + + [CounterMetric] + [Description("Number of pending transactions received that were ignored because of fee lower than the lowest fee in transaction pool.")] public static long PendingTransactionsTooLowFee { get; set; } [CounterMetric] From 29f73f19ea242761cc9c619f1a723b4f216bd17d Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:09:06 +0200 Subject: [PATCH 139/148] adjust TxPool filters pipeline and TxPoolReport --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 33 ++++++++++++---------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index 4fc92ef11c5..ff933e53130 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -112,6 +112,7 @@ public TxPool(IEthereumEcdsa ecdsa, { new NotSupportedTxFilter(txPoolConfig, _logger), new GasLimitTxFilter(_headInfo, txPoolConfig, _logger), + new PriorityFeeTooLowFilter(_logger), new FeeTooLowFilter(_headInfo, _transactions, _blobTransactions, thereIsPriorityContract, _logger), new MalformedTxFilter(_specProvider, validator, _logger) }; @@ -716,8 +717,9 @@ private static void WriteTxPoolReport(ILogger logger) if (float.IsNaN(receivedDiscarded)) receivedDiscarded = 0; logger.Info(@$" -Txn Pool State ({Metrics.TransactionCount:N0} txns queued) -Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) +------------------------------------------------ +TxPool: {Metrics.TransactionCount:N0} txns queued +BlobPool: {Metrics.BlobTransactionCount:N0} txns queued ------------------------------------------------ Sent * Transactions: {Metrics.PendingTransactionsSent,24:N0} @@ -730,19 +732,20 @@ Blob txs Pool ({Metrics.BlobTransactionCount:N0} txns queued) Discarded at Filter Stage: 1. NotSupportedTxType {Metrics.PendingTransactionsNotSupportedTxType,24:N0} 2. GasLimitTooHigh: {Metrics.PendingTransactionsGasLimitTooHigh,24:N0} -3. Too Low Fee: {Metrics.PendingTransactionsTooLowFee,24:N0} -4. Malformed {Metrics.PendingTransactionsMalformed,24:N0} -5. Duplicate: {Metrics.PendingTransactionsKnown,24:N0} -6. Unknown Sender: {Metrics.PendingTransactionsUnresolvableSender,24:N0} -7. Conflicting TxType {Metrics.PendingTransactionsConflictingTxType,24:N0} -8. NonceTooFarInFuture {Metrics.PendingTransactionsNonceTooFarInFuture,24:N0} -9. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} -10. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} -11. Balance Too Low: {Metrics.PendingTransactionsTooLowBalance,24:N0} -12. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} -13. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} -14. Failed replacement {Metrics.PendingTransactionsPassedFiltersButCannotReplace,24:N0} -15. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} +3. TooLow PriorityFee: {Metrics.PendingTransactionsTooLowPriorityFee,24:N0} +4. Too Low Fee: {Metrics.PendingTransactionsTooLowFee,24:N0} +5. Malformed {Metrics.PendingTransactionsMalformed,24:N0} +6. Duplicate: {Metrics.PendingTransactionsKnown,24:N0} +7. Unknown Sender: {Metrics.PendingTransactionsUnresolvableSender,24:N0} +8. Conflicting TxType {Metrics.PendingTransactionsConflictingTxType,24:N0} +9. NonceTooFarInFuture {Metrics.PendingTransactionsNonceTooFarInFuture,24:N0} +10. Zero Balance: {Metrics.PendingTransactionsZeroBalance,24:N0} +11. Balance < tx.value: {Metrics.PendingTransactionsBalanceBelowValue,24:N0} +12. Balance Too Low: {Metrics.PendingTransactionsTooLowBalance,24:N0} +13. Nonce used: {Metrics.PendingTransactionsLowNonce,24:N0} +14. Nonces skipped: {Metrics.PendingTransactionsNonceGap,24:N0} +15. Failed replacement {Metrics.PendingTransactionsPassedFiltersButCannotReplace,24:N0} +16. Cannot Compete: {Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees,24:N0} ------------------------------------------------ Validated via State: {Metrics.PendingTransactionsWithExpensiveFiltering,24:N0} ------------------------------------------------ From 2df4ae2bcd63edc765a9229b6d3168ca6b51e040 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:12:48 +0200 Subject: [PATCH 140/148] change naming as requested in review --- .../Nethermind.Blockchain.Test/TransactionSelectorTests.cs | 2 +- src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs | 2 +- src/Nethermind/Nethermind.TxPool/ITxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/NullTxPool.cs | 2 +- src/Nethermind/Nethermind.TxPool/TxPool.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index 452b6505da7..dd34a674deb 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -245,7 +245,7 @@ void SetAccountStates(IEnumerable
missingAddresses) g => g.Key!, g => g.OrderBy(t => t, comparer).ToArray()); transactionPool.GetPendingTransactionsBySender().Returns(transactions); - transactionPool.GetPendingBlobTransactionsEquivalencesBySender().Returns(blobTransactions); + transactionPool.GetPendingLightBlobTransactionsBySender().Returns(blobTransactions); foreach (KeyValuePair keyValuePair in blobTransactions) { foreach (Transaction blobTx in keyValuePair.Value) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 186889eb521..480386fb30f 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -50,7 +50,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi IEip1559Spec specFor1559 = _specProvider.GetSpecFor1559(blockNumber); UInt256 baseFee = BaseFeeCalculator.Calculate(parent, specFor1559); IDictionary pendingTransactions = _transactionPool.GetPendingTransactionsBySender(); - IDictionary pendingBlobTransactionsEquivalences = _transactionPool.GetPendingBlobTransactionsEquivalencesBySender(); + IDictionary pendingBlobTransactionsEquivalences = _transactionPool.GetPendingLightBlobTransactionsBySender(); IComparer comparer = GetComparer(parent, new BlockPreparationContext(baseFee, blockNumber)) .ThenBy(ByHashTxComparer.Instance); // in order to sort properly and not lose transactions we need to differentiate on their identity which provided comparer might not be doing diff --git a/src/Nethermind/Nethermind.TxPool/ITxPool.cs b/src/Nethermind/Nethermind.TxPool/ITxPool.cs index f3392ee6ae4..91d8d725a1a 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPool.cs @@ -26,7 +26,7 @@ public interface ITxPool /// Blob txs light equivalences grouped by sender address, sorted by nonce and later tx pool sorting ///
/// - IDictionary GetPendingBlobTransactionsEquivalencesBySender(); + IDictionary GetPendingLightBlobTransactionsBySender(); /// /// from a specific sender, sorted by nonce and later tx pool sorting diff --git a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs index 7944ea06cd8..760f3ea29d6 100644 --- a/src/Nethermind/Nethermind.TxPool/NullTxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/NullTxPool.cs @@ -25,7 +25,7 @@ private NullTxPool() { } public IDictionary GetPendingTransactionsBySender() => new Dictionary(); - public IDictionary GetPendingBlobTransactionsEquivalencesBySender() + public IDictionary GetPendingLightBlobTransactionsBySender() => new Dictionary(); public IEnumerable GetPendingBlobTransactions() => Array.Empty(); diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index ff933e53130..f15d2a37249 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -158,7 +158,7 @@ public TxPool(IEthereumEcdsa ecdsa, public IDictionary GetPendingTransactionsBySender() => _transactions.GetBucketSnapshot(); - public IDictionary GetPendingBlobTransactionsEquivalencesBySender() => + public IDictionary GetPendingLightBlobTransactionsBySender() => _blobTransactions.GetBucketSnapshot(); public Transaction[] GetPendingTransactionsBySender(Address address) => From d90e4a26de6db241f09e6257ad3135ee071877cb Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:27:24 +0200 Subject: [PATCH 141/148] add size estimates to descriptions in ITxPoolConfig --- src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs | 4 ++-- src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs index 4e3dcd52bc1..9f7d9ef9d0b 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxPoolConfig.cs @@ -19,10 +19,10 @@ public interface ITxPoolConfig : IConfig [ConfigItem(DefaultValue = "false", Description = "If true, all blob transactions would be stored in persistent db")] bool PersistentBlobStorageEnabled { get; set; } - [ConfigItem(DefaultValue = "16384", Description = "Max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage)")] + [ConfigItem(DefaultValue = "16384", Description = "Max number of full blob transactions stored in the database (increasing the number of transactions in the blob pool also results in higher memory usage). Default value use max 13GB (16386*128KB*6blobs), for 1-blob txs it's 2GB (16386*128KB)")] int PersistentBlobStorageSize { get; set; } - [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage")] + [ConfigItem(DefaultValue = "256", Description = "Max number of full blob transactions stored in memory as a cache for persistent storage. Default value use max 200MB (256*128KB*6blobs), for 1-blob txs it's 33MB (256*128KB)")] int BlobCacheSize { get; set; } [ConfigItem(DefaultValue = "512", Description = "Max number of full blob transactions stored in memory. Used only if persistent storage is disabled")] diff --git a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs index e6975cbadb2..071ed94dd04 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPoolConfig.cs @@ -9,7 +9,7 @@ public class TxPoolConfig : ITxPoolConfig public int Size { get; set; } = 2048; public bool BlobSupportEnabled { get; set; } = false; public bool PersistentBlobStorageEnabled { get; set; } = false; - public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 12,5GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); + public int PersistentBlobStorageSize { get; set; } = 16 * 1024; // theoretical max - 13GB (128KB * 6 * 16384); for one-blob txs - 2GB (128KB * 1 * 16384); // practical max - something between, but closer to 2GB than 12GB. Geth is limiting it to 10GB. // every day about 21600 blobs will be included (7200 blocks per day * 3 blob target) public int BlobCacheSize { get; set; } = 256; From 544dc68eeac2f84cc98829706d6901230a55048f Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:28:03 +0200 Subject: [PATCH 142/148] rename AcceptTxResult PendingTxsOfOtherType -> PendingTxsOfConflictingType --- src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs | 2 +- src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs | 2 +- src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs index d1a3bfa140c..a8f2d8c241e 100644 --- a/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs +++ b/src/Nethermind/Nethermind.TxPool.Test/TxPoolTests.Blobs.cs @@ -253,7 +253,7 @@ Transaction GetTx(bool isBlob, UInt256 nonce) Transaction secondTx = GetTx(secondIsBlob, UInt256.One); _txPool.SubmitTx(firstTx, TxHandlingOptions.None).Should().Be(AcceptTxResult.Accepted); - _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfOtherType : AcceptTxResult.Accepted); + _txPool.SubmitTx(secondTx, TxHandlingOptions.None).Should().Be(firstIsBlob ^ secondIsBlob ? AcceptTxResult.PendingTxsOfConflictingType : AcceptTxResult.Accepted); } [Test] diff --git a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs index 04a5fec68bf..f8a0033dfe7 100644 --- a/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs +++ b/src/Nethermind/Nethermind.TxPool/AcceptTxResult.cs @@ -83,7 +83,7 @@ namespace Nethermind.TxPool /// /// Ignores blob transactions if sender already have pending transactions of other types; ignore other types if has already pending blobs /// - public static readonly AcceptTxResult PendingTxsOfOtherType = new(14, nameof(PendingTxsOfOtherType)); + public static readonly AcceptTxResult PendingTxsOfConflictingType = new(14, nameof(PendingTxsOfConflictingType)); /// /// Ignores transactions if tx type is not supported diff --git a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs index 075503b96cc..be79182e8c7 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/TxTypeTxFilter.cs @@ -26,7 +26,7 @@ public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingO if (otherTxTypePool.ContainsBucket(tx.SenderAddress!)) // as unknownSenderFilter will run before this one { Metrics.PendingTransactionsConflictingTxType++; - return AcceptTxResult.PendingTxsOfOtherType; + return AcceptTxResult.PendingTxsOfConflictingType; } return AcceptTxResult.Accepted; } From 9700c48160ebe9569b14859aca58fb7a9aae7b45 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 11:48:35 +0200 Subject: [PATCH 143/148] add more comments to TxPool --- src/Nethermind/Nethermind.TxPool/TxPool.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/TxPool.cs b/src/Nethermind/Nethermind.TxPool/TxPool.cs index f15d2a37249..944833fb757 100644 --- a/src/Nethermind/Nethermind.TxPool/TxPool.cs +++ b/src/Nethermind/Nethermind.TxPool/TxPool.cs @@ -394,15 +394,19 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe if (!inserted) { + // it means it failed on adding to the pool - it is possible when new tx has the same sender + // and nonce as already existent tx and is not good enough to replace it Metrics.PendingTransactionsPassedFiltersButCannotReplace++; return AcceptTxResult.ReplacementNotAllowed; } if (tx.Hash == removed?.Hash) { + // it means it was added and immediately evicted - pool was full of better txs if (isPersistentBroadcast) { - // it means it was added and immediately evicted - we are adding only to persistent broadcast + // we are adding only to persistent broadcast - not good enough for standard pool, + // but can be good enough for TxBroadcaster pool - for local txs only _broadcaster.Broadcast(tx, isPersistentBroadcast); } Metrics.PendingTransactionsPassedFiltersButCannotCompeteOnFees++; @@ -470,6 +474,7 @@ private AcceptTxResult AddCore(Transaction tx, TxFilteringState state, bool isPe previousTxBottleneck ??= tx.CalculateAffordableGasPrice(_specProvider.GetCurrentHeadSpec().IsEip1559Enabled, _headInfo.CurrentBaseFee, balance); + // it is not affecting non-blob txs - for them MaxFeePerBlobGas is null so check is skipped if (tx.MaxFeePerBlobGas < _headInfo.CurrentPricePerBlobGas) { gasBottleneck = UInt256.Zero; From 222cd927bf04b99b937d53b2b41ff895137c5863 Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 12:08:23 +0200 Subject: [PATCH 144/148] add more comments to PersistentBlobTxDistinctSortedPool --- .../Collections/PersistentBlobTxDistinctSortedPool.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs index 452000450ff..419e8a3e68d 100644 --- a/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs +++ b/src/Nethermind/Nethermind.TxPool/Collections/PersistentBlobTxDistinctSortedPool.cs @@ -65,15 +65,20 @@ public override bool TryInsert(ValueKeccak hash, Transaction fullBlobTx, out Tra public override bool TryGetValue(ValueKeccak hash, [NotNullWhen(true)] out Transaction? fullBlobTx) { + // Firstly check if tx is present in in-memory collection of light blob txs (without actual blobs). + // If not, just return false if (base.TryGetValue(hash, out Transaction? lightTx)) { + // tx is present in light collection. Try to get full blob tx from cache if (_blobTxCache.TryGet(hash, out fullBlobTx)) { return true; } + // tx is present, but not cached, at this point we need to load it from db... if (_blobTxStorage.TryGet(hash, lightTx.SenderAddress!, lightTx.Timestamp, out fullBlobTx)) { + // ...and we are saving recently used blob tx to cache _blobTxCache.Set(hash, fullBlobTx); return true; } From 4e26565dc6172a7472a5898eb936d40ef17dbaed Mon Sep 17 00:00:00 2001 From: Marcin Sobczak Date: Tue, 10 Oct 2023 13:24:15 +0200 Subject: [PATCH 145/148] fix file encoding --- .../Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs index 3b455788ef7..34003b0bdfc 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs @@ -1,4 +1,4 @@ -// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited +// SPDX-FileCopyrightText: 2023 Demerzel Solutions Limited // SPDX-License-Identifier: LGPL-3.0-only using System.Diagnostics; From fb97b145bd5971c5f47adda7842da683f6a77a48 Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Wed, 11 Oct 2023 12:10:33 +0200 Subject: [PATCH 146/148] small refactors --- .../TransactionSelectorTests.cs | 43 +++++++++---------- .../StandardDbInitializerTests.cs | 12 +++--- .../Nethermind.TxPool/BlobTxStorage.cs | 10 +++-- .../Nethermind.TxPool/ITxStorage.cs | 4 +- .../Nethermind.TxPool/LightTransaction.cs | 8 +++- .../Nethermind.TxPool/NullBlobTxStorage.cs | 4 +- 6 files changed, 43 insertions(+), 38 deletions(-) diff --git a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs index dd34a674deb..3b3b7ef1198 100644 --- a/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs +++ b/src/Nethermind/Nethermind.Blockchain.Test/TransactionSelectorTests.cs @@ -230,34 +230,30 @@ void SetAccountStates(IEnumerable
missingAddresses) new(specProvider, blockTree); IComparer defaultComparer = transactionComparerProvider.GetDefaultComparer(); IComparer comparer = CompareTxByNonce.Instance.ThenBy(defaultComparer); - Dictionary transactions = testCase.Transactions - .Where(t => t.SenderAddress is not null) - .Where(t => !t.SupportsBlobs) - .GroupBy(t => t.SenderAddress) - .ToDictionary( - g => g.Key!, - g => g.OrderBy(t => t, comparer).ToArray()); - Dictionary blobTransactions = testCase.Transactions - .Where(t => t?.SenderAddress is not null) - .Where(t => t.SupportsBlobs) - .GroupBy(t => t.SenderAddress) - .ToDictionary( - g => g.Key!, - g => g.OrderBy(t => t, comparer).ToArray()); + + Dictionary GroupTransactions(bool supportBlobs) => + testCase.Transactions + .Where(t => t.SenderAddress is not null) + .Where(t => t.SupportsBlobs == supportBlobs) + .GroupBy(t => t.SenderAddress) + .ToDictionary( + g => g.Key!, + g => g.OrderBy(t => t, comparer).ToArray()); + + Dictionary transactions = GroupTransactions(false); + Dictionary blobTransactions = GroupTransactions(true); transactionPool.GetPendingTransactionsBySender().Returns(transactions); transactionPool.GetPendingLightBlobTransactionsBySender().Returns(blobTransactions); - foreach (KeyValuePair keyValuePair in blobTransactions) + foreach (Transaction blobTx in blobTransactions.SelectMany(kvp => kvp.Value)) { - foreach (Transaction blobTx in keyValuePair.Value) + transactionPool.TryGetPendingBlobTransaction(Arg.Is(h => h == blobTx.Hash), + out Arg.Any()).Returns(x => { - transactionPool.TryGetPendingBlobTransaction(Arg.Is(h => h == blobTx.Hash), - out Arg.Any()).Returns(x => - { - x[1] = blobTx; - return true; - }); - } + x[1] = blobTx; + return true; + }); } + BlocksConfig blocksConfig = new() { MinGasPrice = testCase.MinGasPriceForMining }; ITxFilterPipeline txFilterPipeline = new TxFilterPipelineBuilder(LimboLogs.Instance) .WithMinGasPriceFilter(blocksConfig, specProvider) @@ -274,6 +270,7 @@ void SetAccountStates(IEnumerable
missingAddresses) { parentHeader = parentHeader.WithExcessBlobGas(0); } + IEnumerable selectedTransactions = poolTxSource.GetTransactions(parentHeader.TestObject, testCase.GasLimit); diff --git a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs index 65fb1da0d0f..74d9d66ab20 100644 --- a/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs +++ b/src/Nethermind/Nethermind.Db.Test/StandardDbInitializerTests.cs @@ -18,7 +18,7 @@ namespace Nethermind.Db.Test [Parallelizable(ParallelScope.All)] public class StandardDbInitializerTests { - private string _folderWithDbs; + private string _folderWithDbs = null!; [OneTimeSetUp] public void Initialize() @@ -30,7 +30,7 @@ public void Initialize() [TestCase(true)] public async Task InitializerTests_MemDbProvider(bool useReceipts) { - IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Mem, "mem"); + using IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Mem, "mem"); Type receiptsType = GetReceiptsType(useReceipts, typeof(MemColumnsDb)); AssertStandardDbs(dbProvider, typeof(MemDb), receiptsType); dbProvider.StateDb.Should().BeOfType(typeof(FullPruningDb)); @@ -40,7 +40,7 @@ public async Task InitializerTests_MemDbProvider(bool useReceipts) [TestCase(true)] public async Task InitializerTests_RocksDbProvider(bool useReceipts) { - IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Persisted, $"rocks_{useReceipts}"); + using IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Persisted, $"rocks_{useReceipts}"); Type receiptsType = GetReceiptsType(useReceipts); AssertStandardDbs(dbProvider, typeof(DbOnTheRocks), receiptsType); dbProvider.StateDb.Should().BeOfType(typeof(FullPruningDb)); @@ -50,7 +50,7 @@ public async Task InitializerTests_RocksDbProvider(bool useReceipts) [TestCase(true)] public async Task InitializerTests_ReadonlyDbProvider(bool useReceipts) { - IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Persisted, $"readonly_{useReceipts}"); + using IDbProvider dbProvider = await InitializeStandardDb(useReceipts, DbModeHint.Persisted, $"readonly_{useReceipts}"); using ReadOnlyDbProvider readonlyDbProvider = new(dbProvider, true); Type receiptsType = GetReceiptsType(useReceipts); AssertStandardDbs(dbProvider, typeof(DbOnTheRocks), receiptsType); @@ -62,13 +62,13 @@ public async Task InitializerTests_ReadonlyDbProvider(bool useReceipts) [Test] public async Task InitializerTests_WithPruning() { - IDbProvider dbProvider = await InitializeStandardDb(false, DbModeHint.Mem, "pruning"); + using IDbProvider dbProvider = await InitializeStandardDb(false, DbModeHint.Mem, "pruning"); dbProvider.StateDb.Should().BeOfType(); } private async Task InitializeStandardDb(bool useReceipts, DbModeHint dbModeHint, string path) { - using IDbProvider dbProvider = new DbProvider(dbModeHint); + IDbProvider dbProvider = new DbProvider(dbModeHint); RocksDbFactory rocksDbFactory = new(new DbConfig(), LimboLogs.Instance, Path.Combine(_folderWithDbs, path)); StandardDbInitializer initializer = new(dbProvider, rocksDbFactory, new MemDbFactory(), Substitute.For()); await initializer.InitStandardDbsAsync(useReceipts); diff --git a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs index 03fd6f267bf..803bb8bccab 100644 --- a/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/BlobTxStorage.cs @@ -5,6 +5,7 @@ using System.Buffers.Binary; using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; +using DotNetty.Buffers; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Db; @@ -32,7 +33,7 @@ public BlobTxStorage(IColumnsDb database) _lightBlobTxsDb = database.GetColumnDb(BlobTxsColumns.LightBlobTxs); } - public bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) + public bool TryGet(in ValueKeccak hash, Address sender, in UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) { Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); @@ -63,14 +64,15 @@ public void Add(Transaction transaction) GetHashPrefixedByTimestamp(transaction.Timestamp, transaction.Hash, txHashPrefixed); int length = _txDecoder.GetLength(transaction, RlpBehaviors.InMempoolForm); - RlpStream rlpStream = new(length); + IByteBuffer byteBuffer = PooledByteBufferAllocator.Default.Buffer(length); + using NettyRlpStream rlpStream = new(byteBuffer); rlpStream.Encode(transaction, RlpBehaviors.InMempoolForm); - _fullBlobTxsDb.Set(txHashPrefixed, rlpStream.Data!); + _fullBlobTxsDb.Set(txHashPrefixed, byteBuffer.AsSpan()); _lightBlobTxsDb.Set(transaction.Hash, _lightTxDecoder.Encode(transaction)); } - public void Delete(ValueKeccak hash, UInt256 timestamp) + public void Delete(in ValueKeccak hash, in UInt256 timestamp) { Span txHashPrefixed = stackalloc byte[64]; GetHashPrefixedByTimestamp(timestamp, hash, txHashPrefixed); diff --git a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs index 445de45ff75..9f7c3f170d3 100644 --- a/src/Nethermind/Nethermind.TxPool/ITxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/ITxStorage.cs @@ -11,8 +11,8 @@ namespace Nethermind.TxPool; public interface ITxStorage { - bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction); + bool TryGet(in ValueKeccak hash, Address sender, in UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction); IEnumerable GetAll(); void Add(Transaction transaction); - void Delete(ValueKeccak hash, UInt256 timestamp); + void Delete(in ValueKeccak hash, in UInt256 timestamp); } diff --git a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs index 1c9cb8d0418..b8dd4f0e416 100644 --- a/src/Nethermind/Nethermind.TxPool/LightTransaction.cs +++ b/src/Nethermind/Nethermind.TxPool/LightTransaction.cs @@ -2,6 +2,8 @@ // SPDX-License-Identifier: LGPL-3.0-only +using System.Collections.Generic; +using System.Linq; using Nethermind.Core; using Nethermind.Core.Crypto; using Nethermind.Int256; @@ -13,6 +15,10 @@ namespace Nethermind.TxPool; ///
public class LightTransaction : Transaction { + private static readonly Dictionary _blobVersionedHashesCache = + Enumerable.Range(1, Eip4844Constants.MaxBlobsPerBlock).ToDictionary(i => i, i => new byte[i][]); + + public LightTransaction(Transaction fullTx) { Type = TxType.Blob; @@ -24,7 +30,7 @@ public LightTransaction(Transaction fullTx) GasPrice = fullTx.GasPrice; // means MaxPriorityFeePerGas DecodedMaxFeePerGas = fullTx.DecodedMaxFeePerGas; MaxFeePerBlobGas = fullTx.MaxFeePerBlobGas; - BlobVersionedHashes = new byte[fullTx.BlobVersionedHashes!.Length][]; + BlobVersionedHashes = _blobVersionedHashesCache[fullTx.BlobVersionedHashes!.Length]; GasBottleneck = fullTx.GasBottleneck; Timestamp = fullTx.Timestamp; PoolIndex = fullTx.PoolIndex; diff --git a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs index 0969732e816..9d0982b882a 100644 --- a/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs +++ b/src/Nethermind/Nethermind.TxPool/NullBlobTxStorage.cs @@ -14,7 +14,7 @@ public class NullBlobTxStorage : ITxStorage { public static NullBlobTxStorage Instance { get; } = new(); - public bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) + public bool TryGet(in ValueKeccak hash, Address sender, in UInt256 timestamp, [NotNullWhen(true)] out Transaction? transaction) { transaction = default; return false; @@ -24,5 +24,5 @@ public bool TryGet(ValueKeccak hash, Address sender, UInt256 timestamp, [NotNull public void Add(Transaction transaction) { } - public void Delete(ValueKeccak hash, UInt256 timestamp) { } + public void Delete(in ValueKeccak hash, in UInt256 timestamp) { } } From 112fb91457a7d340b3c1d94cdd1695879b58719b Mon Sep 17 00:00:00 2001 From: MarekM25 Date: Wed, 11 Oct 2023 12:34:07 +0200 Subject: [PATCH 147/148] cosmetic 1.GWei() --- .../Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs index 34003b0bdfc..8026c49808a 100644 --- a/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs +++ b/src/Nethermind/Nethermind.TxPool/Filters/PriorityFeeTooLowFilter.cs @@ -3,6 +3,8 @@ using System.Diagnostics; using Nethermind.Core; +using Nethermind.Core.Extensions; +using Nethermind.Int256; using Nethermind.Logging; namespace Nethermind.TxPool.Filters; @@ -10,7 +12,7 @@ namespace Nethermind.TxPool.Filters; public class PriorityFeeTooLowFilter : IIncomingTxFilter { private readonly ILogger _logger; - private const int OneGWei = 1_000_000_000; + private static readonly UInt256 _minBlobsPriorityFee = 1.GWei(); public PriorityFeeTooLowFilter(ILogger logger) { @@ -19,11 +21,11 @@ public PriorityFeeTooLowFilter(ILogger logger) public AcceptTxResult Accept(Transaction tx, TxFilteringState state, TxHandlingOptions handlingOptions) { - if (tx.SupportsBlobs && tx.MaxPriorityFeePerGas < OneGWei) + if (tx.SupportsBlobs && tx.MaxPriorityFeePerGas < _minBlobsPriorityFee) { Metrics.PendingTransactionsTooLowPriorityFee++; if (_logger.IsTrace) _logger.Trace($"Skipped adding transaction {tx.ToString(" ")}, too low priority fee with options {handlingOptions} from {new StackTrace()}"); - return AcceptTxResult.FeeTooLow.WithMessage($"MaxPriorityFeePerGas for blob transaction needs to be at least {OneGWei} (1 GWei), is {tx.MaxPriorityFeePerGas}."); + return AcceptTxResult.FeeTooLow.WithMessage($"MaxPriorityFeePerGas for blob transaction needs to be at least {_minBlobsPriorityFee} (1 GWei), is {tx.MaxPriorityFeePerGas}."); } return AcceptTxResult.Accepted; From 001be21f51db4093774e1088b6ba63e6242ef0eb Mon Sep 17 00:00:00 2001 From: "lukasz.rozmej" Date: Wed, 11 Oct 2023 12:34:48 +0200 Subject: [PATCH 148/148] reduce spec lookup --- .../Producers/TxPoolTxSource.cs | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs index 480386fb30f..7b291568bd5 100644 --- a/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs +++ b/src/Nethermind/Nethermind.Consensus/Producers/TxPoolTxSource.cs @@ -47,8 +47,8 @@ public TxPoolTxSource( public IEnumerable GetTransactions(BlockHeader parent, long gasLimit) { long blockNumber = parent.Number + 1; - IEip1559Spec specFor1559 = _specProvider.GetSpecFor1559(blockNumber); - UInt256 baseFee = BaseFeeCalculator.Calculate(parent, specFor1559); + IReleaseSpec spec = _specProvider.GetSpec(parent); + UInt256 baseFee = BaseFeeCalculator.Calculate(parent, spec); IDictionary pendingTransactions = _transactionPool.GetPendingTransactionsBySender(); IDictionary pendingBlobTransactionsEquivalences = _transactionPool.GetPendingLightBlobTransactionsBySender(); IComparer comparer = GetComparer(parent, new BlockPreparationContext(baseFee, blockNumber)) @@ -62,7 +62,7 @@ public IEnumerable GetTransactions(BlockHeader parent, long gasLimi int selectedTransactions = 0; using ArrayPoolList selectedBlobTxs = new(Eip4844Constants.MaxBlobsPerBlock); - SelectBlobTransactions(blobTransactions, parent, selectedBlobTxs); + SelectBlobTransactions(blobTransactions, parent, spec, selectedBlobTxs); foreach (Transaction tx in transactions) { @@ -117,7 +117,7 @@ private IEnumerable PickBlobTxsBetterThanCurrentTx(ArrayPoolList blobTransactions, BlockHeader parent, ArrayPoolList selectedBlobTxs) + private void SelectBlobTransactions(IEnumerable blobTransactions, BlockHeader parent, IReleaseSpec spec, ArrayPoolList selectedBlobTxs) { int checkedBlobTransactions = 0; int selectedBlobTransactions = 0; @@ -141,7 +141,7 @@ private void SelectBlobTransactions(IEnumerable blobTransactions, B continue; } - if (blobGasPrice.IsZero && !TryUpdateBlobGasPrice(blobTx, parent, out blobGasPrice)) + if (blobGasPrice.IsZero && !TryUpdateBlobGasPrice(blobTx, parent, spec, out blobGasPrice)) { if (_logger.IsTrace) _logger.Trace($"Declining {blobTx.ToShortString()}, failed to get full version of this blob tx from TxPool."); continue; @@ -179,25 +179,28 @@ private bool TryGetFullBlobTx(Transaction blobTx, [NotNullWhen(true)] out Transa fullBlobTx = blobTx; return true; } + fullBlobTx = null; return blobTx.Hash is not null && _transactionPool.TryGetPendingBlobTransaction(blobTx.Hash, out fullBlobTx); } - private bool TryUpdateBlobGasPrice(Transaction lightBlobTx, BlockHeader parent, out UInt256 blobGasPrice) + private bool TryUpdateBlobGasPrice(Transaction lightBlobTx, BlockHeader parent, IReleaseSpec spec, out UInt256 blobGasPrice) { - ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, _specProvider.GetSpec(parent)); + ulong? excessDataGas = BlobGasCalculator.CalculateExcessBlobGas(parent, spec); if (excessDataGas is null) { if (_logger.IsTrace) _logger.Trace($"Declining {lightBlobTx.ToShortString()}, the specification is not configured to handle shard blob transactions."); blobGasPrice = UInt256.Zero; return false; } + if (!BlobGasCalculator.TryCalculateBlobGasPricePerUnit(excessDataGas.Value, out blobGasPrice)) { if (_logger.IsTrace) _logger.Trace($"Declining {lightBlobTx.ToShortString()}, failed to calculate data gas price."); blobGasPrice = UInt256.Zero; return false; } + return true; }