Erigon v2.60.1 (#403)

* fix Consensus specification tests CI (erigontech#10391) (erigontech#10396)

Cherry-pick:
erigontech@bc5fa6f

Need this to get PR CI green for v2.60.1 patches, e.g. -
erigontech#10390

Co-authored-by: Andrew Ashikhmin <[email protected]>

* rpc/handler: do not append null to stream when json may be valid (erigontech#10390)

Cherry-pick:
erigontech@4d1c954
Relates to: erigontech#10376

* Fixed Bor Log appearing on Ethereum Mainnet (erigontech#10405) (erigontech#10420)

Cherry-pick:
erigontech@be889f6

Co-authored-by: Giulio rebuffo <[email protected]>

* fix gas price not right problem (erigontech#10456)

Cherry pick PR erigontech#10451 into the release branch

Co-authored-by: mars <[email protected]>

* eth_estimateGas: default feeCap to base fee (erigontech#10499)

Copy PR erigontech#10495 into the release branch

* Add flag for bor waypoint types (erigontech#10501)

Cherry pick PR erigontech#10281 into the release branch

Co-authored-by: Mark Holt <[email protected]>
Co-authored-by: alex.sharov <[email protected]>

* try to fix 'method handler crashed' for debug_traceCall of erigontech#9090 (erigontech#10502)

Cherry pick PR erigontech#10401 into the release branch

Co-authored-by: mars <[email protected]>

* diagnostics: cherry pick speedtest disable (erigontech#10509)

Cherry pick PR erigontech#10449 into the release branch

* Enable DNS p2p discovery on holesky (erigontech#10507)

Cherry pick PR erigontech#10460 into the release branch

Co-authored-by: Willian Mitsuda <[email protected]>

* fix eth_call 'method handler crashed' error when tx has set maxFeePerBlobGas (erigontech#10506)

Cherry pick PR erigontech#10452 into the release branch

Co-authored-by: mars <[email protected]>

* e2: remove overlapped files only after merge (erigontech#10487)

Otherwise: if start after `kill -9` in the middle of merge - may remove
small files of 1 type of file, but leave small files of another type of
files (which merge was not finished) - and leave node in un-mergable
state: erigontech#10485

---------

Co-authored-by: awskii <[email protected]>

* add flag checking for pruning waypoints (erigontech#10508)

Cherry pick PR erigontech#10468 into the release branch

Co-authored-by: Mark Holt <[email protected]>

* p2p/sentry: sentry doesn't start with ErrNoHead (erigontech#10454) (erigontech#10523)

cherry-pick erigontech#10494 to
release/2.60

* add lock to purgeMilestoneIDsList (erigontech#10524)

Cherry pick PR erigontech#10493 into the release branch

Co-authored-by: Mark Holt <[email protected]>

* polygon/heimdall: fix checkpoint json marshalling (erigontech#10530)

Fixes a recent regression causing unwinds due to checkpoints having zero
root hash:
```
[WARN] [05-18|23:58:54.662] [bor] Root hash mismatch while whitelisting checkpoint expected=ac1c57270479250af3ce8eee90075cd8b2ba1bac55353105e063d9a4c87c743e got=0000000000000000000000000000000000000000000000000000000000000000
[WARN] [05-18|23:58:54.662] [bor] Rewinding chain due to checkpoint root hash mismatch number=57125727
```

Note this has already been fixed on Erigon 3 branch but as part of a
non-related PR -
https://github.com/ledgerwatch/erigon/pull/10124/files#diff-47d4532f399f2d6a45e6f19944a45c80bac573b4d1b5cb51485d0254229d1b16

* Fix capacity for immediate appends (erigontech#10539)

Cherry pick PR erigontech#10528 into the release branch

Co-authored-by: Shoham Chakraborty <[email protected]>

* core/vm: set tracer-observable value of a delegatecall to match parent value (erigontech#10370)

requested by erigontech#9549

port of ethereum/go-ethereum#26632

* params: version 2.60.1 (erigontech#10555)

* blobGasPrice should be marshalled as hex (erigontech#10571)

Cherry pick PR erigontech#10551 into the release branch

* Caplin: Fixed reforwarding of Bls Execution changes (erigontech#10577)

Cherry pick PR erigontech#10546 into the release branch

Co-authored-by: Giulio rebuffo <[email protected]>

* Caplin: Proper "Normalization" of length of ForkVersions to 8 hex characters (erigontech#10578)

Cherry pick PR erigontech#10512 into the release branch

Co-authored-by: Giulio rebuffo <[email protected]>

* Caplin: Update BlobSidecars Beacon API endpoint to the latest specs (erigontech#10580)

Cherry pick PR erigontech#10576 into the release branch

Co-authored-by: Giulio rebuffo <[email protected]>

* bor blocks retire: infinity loop fix (erigontech#10596)

Problem: `+1` was added to maxBlockNum instead of minBlockNum
for: erigontech#10554

* txpool: EIP-3860 should only apply to create transactions (erigontech#10609)

This fixes Issue erigontech#10607

---------

Co-authored-by: milen <[email protected]>
Co-authored-by: Andrew Ashikhmin <[email protected]>
Co-authored-by: Giulio rebuffo <[email protected]>
Co-authored-by: mars <[email protected]>
Co-authored-by: Mark Holt <[email protected]>
Co-authored-by: alex.sharov <[email protected]>
Co-authored-by: Dmytro <[email protected]>
Co-authored-by: Willian Mitsuda <[email protected]>
Co-authored-by: awskii <[email protected]>
Co-authored-by: battlmonstr <[email protected]>
Co-authored-by: Shoham Chakraborty <[email protected]>

Author: 12 people

cl/beacon/beaconhttp/args.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"regexp"
 	"strconv"
+	"strings"
 
 	"github.com/go-chi/chi/v5"
 	"github.com/ledgerwatch/erigon-lib/common"
@@ -168,9 +169,13 @@ func Uint64FromQueryParams(r *http.Request, name string) (*uint64, error) {
 
 // decode a list of strings from the query params
 func StringListFromQueryParams(r *http.Request, name string) ([]string, error) {
-	str := r.URL.Query().Get(name)
-	if str == "" {
+	values := r.URL.Query()[name]
+	if len(values) == 0 {
 		return nil, nil
 	}
+
+	// Combine all values into a single string, separating by comma
+	str := strings.Join(values, ",")
+
 	return regexp.MustCompile(`\s*,\s*`).Split(str, -1), nil
 }

cl/beacon/handler/blobs.go

@@ -3,6 +3,7 @@ package handler
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 
 	"github.com/ledgerwatch/erigon/cl/beacon/beaconhttp"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -51,13 +52,33 @@ func (a *ApiHandler) GetEthV1BeaconBlobSidecars(w http.ResponseWriter, r *http.R
 	if err != nil {
 		return nil, err
 	}
-
+	strIdxs, err := beaconhttp.StringListFromQueryParams(r, "indices")
+	if err != nil {
+		return nil, err
+	}
 	resp := solid.NewStaticListSSZ[*cltypes.BlobSidecar](696969, blobSidecarSSZLenght)
 	if !found {
 		return beaconhttp.NewBeaconResponse(resp), nil
 	}
-	for _, v := range out {
-		resp.Append(v)
+	if len(strIdxs) == 0 {
+		for _, v := range out {
+			resp.Append(v)
+		}
+	} else {
+		included := make(map[uint64]struct{})
+		for _, idx := range strIdxs {
+			i, err := strconv.ParseUint(idx, 10, 64)
+			if err != nil {
+				return nil, err
+			}
+			included[i] = struct{}{}
+		}
+		for _, v := range out {
+			if _, ok := included[v.Index]; ok {
+				resp.Append(v)
+			}
+		}
 	}
+
 	return beaconhttp.NewBeaconResponse(resp), nil
 }

cl/clparams/config.go

@@ -15,6 +15,7 @@ package clparams
 
 import (
 	"crypto/rand"
+	"encoding/binary"
 	"fmt"
 	"math"
 	"math/big"
@@ -315,7 +316,9 @@ func (b ConfigByte) MarshalJSON() ([]byte, error) {
 type ConfigForkVersion uint32
 
 func (v ConfigForkVersion) MarshalJSON() ([]byte, error) {
-	return []byte(fmt.Sprintf("\"0x%x\"", v)), nil
+	tmp := make([]byte, 4)
+	binary.BigEndian.PutUint32(tmp, uint32(v))
+	return []byte(fmt.Sprintf("\"0x%x\"", tmp)), nil
 }
 
 // BeaconChainConfig contains constant configs for node to participate in beacon chain.

cl/sentinel/service/service.go

@@ -96,7 +96,8 @@ func (s *SentinelServer) PublishGossip(_ context.Context, msg *sentinelrpc.Gossi
 		gossip.TopicNameVoluntaryExit,
 		gossip.TopicNameProposerSlashing,
 		gossip.TopicNameSyncCommitteeContributionAndProof,
-		gossip.TopicNameAttesterSlashing:
+		gossip.TopicNameAttesterSlashing,
+		gossip.TopicNameBlsToExecutionChange:
 		subscription = manager.GetMatchingSubscription(msg.Name)
 	default:
 		// check subnets

cl/spectest/Makefile

@@ -2,7 +2,7 @@
 
 
 tests:
-	GIT_LFS_SKIP_SMUDGE=1 git clone https://github.com/ethereum/consensus-spec-tests
+	GIT_LFS_SKIP_SMUDGE=1 GIT_CLONE_PROTECTION_ACTIVE=false git clone https://github.com/ethereum/consensus-spec-tests
 	cd consensus-spec-tests && git checkout 080c96fbbf3be58e75947debfeb9ba3b2b7c9748 && git lfs pull --exclude=tests/general,tests/minimal && cd ..
 	mv consensus-spec-tests/tests .
 	rm -rf consensus-spec-tests

cmd/integration/commands/stages.go

@@ -1627,6 +1627,7 @@ func newSync(ctx context.Context, db kv.RwDB, miningConfig *params.MiningConfig,
 		chainConfig,
 		genesisBlock,
 		chainConfig.ChainID.Uint64(),
+		logger,
 	)
 
 	maxBlockBroadcastPeers := func(header *types.Header) uint { return 0 }

cmd/snapshots/manifest/manifest.go

@@ -12,12 +12,13 @@ import (
 	"strings"
 	"time"
 
+	"github.com/urfave/cli/v2"
+
 	"github.com/ledgerwatch/erigon-lib/downloader"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
 	"github.com/ledgerwatch/erigon/cmd/snapshots/sync"
 	"github.com/ledgerwatch/erigon/cmd/utils"
 	"github.com/ledgerwatch/erigon/turbo/logging"
-	"github.com/urfave/cli/v2"
 )
 
 var (
@@ -286,7 +287,7 @@ func verifyManifest(ctx context.Context, srcSession *downloader.RCloneSession, v
 	var extra string
 
 	if len(manifestFiles) != 0 {
-		files := make([]string, len(manifestFiles))
+		files := make([]string, 0, len(manifestFiles))
 
 		for file := range manifestFiles {
 			files = append(files, file)
@@ -296,7 +297,7 @@ func verifyManifest(ctx context.Context, srcSession *downloader.RCloneSession, v
 	}
 
 	if len(dirFiles) != 0 {
-		files := make([]string, len(dirFiles))
+		files := make([]string, 0, len(dirFiles))
 
 		for file := range dirFiles {
 			files = append(files, file)

cmd/utils/flags.go

@@ -61,6 +61,7 @@ import (
 	"github.com/ledgerwatch/erigon/p2p/nat"
 	"github.com/ledgerwatch/erigon/p2p/netutil"
 	"github.com/ledgerwatch/erigon/params"
+	borsnaptype "github.com/ledgerwatch/erigon/polygon/bor/snaptype"
 	"github.com/ledgerwatch/erigon/rpc/rpccfg"
 	"github.com/ledgerwatch/erigon/turbo/logging"
 )
@@ -1024,9 +1025,14 @@ var (
 		Usage: "Diagnostics HTTP server listening port",
 		Value: 6060,
 	}
+	DiagSpeedTestFlag = cli.BoolFlag{
+		Name:  "diagnostics.speedtest",
+		Usage: "Enable speed test",
+		Value: false,
+	}
 )
 
-var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag, &DiagDisabledFlag, &DiagEndpointAddrFlag, &DiagEndpointPortFlag}
+var MetricFlags = []cli.Flag{&MetricsEnabledFlag, &MetricsHTTPFlag, &MetricsPortFlag, &DiagDisabledFlag, &DiagEndpointAddrFlag, &DiagEndpointPortFlag, &DiagSpeedTestFlag}
 
 var DiagnosticsFlags = []cli.Flag{&DiagnosticsURLFlag, &DiagnosticsURLFlag, &DiagnosticsSessionsFlag}
 
@@ -1603,6 +1609,7 @@ func setBorConfig(ctx *cli.Context, cfg *ethconfig.Config) {
 	cfg.WithoutHeimdall = ctx.Bool(WithoutHeimdallFlag.Name)
 	cfg.WithHeimdallMilestones = ctx.Bool(WithHeimdallMilestones.Name)
 	cfg.WithHeimdallWaypointRecording = ctx.Bool(WithHeimdallWaypoints.Name)
+	borsnaptype.RecordWayPoints(cfg.WithHeimdallWaypointRecording)
 	cfg.PolygonSync = ctx.Bool(PolygonSyncFlag.Name)
 }
 

core/blockchain.go

@@ -361,7 +361,7 @@ func logReceipts(receipts types.Receipts, txns types.Transactions, cc *chain.Con
 		return
 	}
 
-	marshalled := make([]map[string]interface{}, len(receipts))
+	marshalled := make([]map[string]interface{}, 0, len(receipts))
 	for i, receipt := range receipts {
 		txn := txns[i]
 		marshalled = append(marshalled, ethutils.MarshalReceipt(receipt, txn, cc, header, txn.Hash(), true))

core/rawdb/accessors_chain.go

@@ -21,7 +21,6 @@ import (
 	"context"
 	"encoding/binary"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"math/big"
@@ -42,7 +41,6 @@ import (
 
 	"github.com/ledgerwatch/erigon/core/types"
 	"github.com/ledgerwatch/erigon/ethdb/cbor"
-	"github.com/ledgerwatch/erigon/polygon/heimdall"
 	"github.com/ledgerwatch/erigon/rlp"
 )
 
@@ -1096,116 +1094,6 @@ func PruneBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int) error {
 	return nil
 }
 
-// PruneBorBlocks - delete [1, to) old blocks after moving it to snapshots.
-// keeps genesis in db: [1, to)
-// doesn't change sequences of kv.EthTx and kv.NonCanonicalTxs
-// doesn't delete Receipts, Senders, Canonical markers, TotalDifficulty
-func PruneBorBlocks(tx kv.RwTx, blockTo uint64, blocksDeleteLimit int, SpanIdAt func(number uint64) uint64) error {
-	c, err := tx.Cursor(kv.BorEventNums)
-	if err != nil {
-		return err
-	}
-	defer c.Close()
-	var blockNumBytes [8]byte
-	binary.BigEndian.PutUint64(blockNumBytes[:], blockTo)
-	k, v, err := c.Seek(blockNumBytes[:])
-	if err != nil {
-		return err
-	}
-	var eventIdTo uint64 = math.MaxUint64
-	if k != nil {
-		eventIdTo = binary.BigEndian.Uint64(v)
-	}
-	c1, err := tx.RwCursor(kv.BorEvents)
-	if err != nil {
-		return err
-	}
-	defer c1.Close()
-	counter := blocksDeleteLimit
-	for k, _, err = c1.First(); err == nil && k != nil && counter > 0; k, _, err = c1.Next() {
-		eventId := binary.BigEndian.Uint64(k)
-		if eventId >= eventIdTo {
-			break
-		}
-		if err = c1.DeleteCurrent(); err != nil {
-			return err
-		}
-		counter--
-	}
-	if err != nil {
-		return err
-	}
-	firstSpanToKeep := SpanIdAt(blockTo)
-	c2, err := tx.RwCursor(kv.BorSpans)
-	if err != nil {
-		return err
-	}
-	defer c2.Close()
-	counter = blocksDeleteLimit
-	for k, _, err := c2.First(); err == nil && k != nil && counter > 0; k, _, err = c2.Next() {
-		spanId := binary.BigEndian.Uint64(k)
-		if spanId >= firstSpanToKeep {
-			break
-		}
-		if err = c2.DeleteCurrent(); err != nil {
-			return err
-		}
-		counter--
-	}
-
-	checkpointCursor, err := tx.RwCursor(kv.BorCheckpoints)
-	if err != nil {
-		return err
-	}
-
-	defer checkpointCursor.Close()
-	lastCheckpointToRemove, err := heimdall.CheckpointIdAt(tx, blockTo)
-
-	if err != nil {
-		return err
-	}
-
-	var checkpointIdBytes [8]byte
-	binary.BigEndian.PutUint64(checkpointIdBytes[:], uint64(lastCheckpointToRemove))
-	for k, _, err := checkpointCursor.Seek(checkpointIdBytes[:]); err == nil && k != nil; k, _, err = checkpointCursor.Prev() {
-		if err = checkpointCursor.DeleteCurrent(); err != nil {
-			return err
-		}
-	}
-
-	milestoneCursor, err := tx.RwCursor(kv.BorMilestones)
-
-	if err != nil {
-		return err
-	}
-
-	defer milestoneCursor.Close()
-
-	var lastMilestoneToRemove heimdall.MilestoneId
-
-	for blockCount := 1; err != nil && blockCount < blocksDeleteLimit; blockCount++ {
-		lastMilestoneToRemove, err = heimdall.MilestoneIdAt(tx, blockTo-uint64(blockCount))
-
-		if !errors.Is(err, heimdall.ErrMilestoneNotFound) {
-			return err
-		} else {
-			if blockCount == blocksDeleteLimit-1 {
-				return nil
-			}
-		}
-	}
-
-	var milestoneIdBytes [8]byte
-	binary.BigEndian.PutUint64(milestoneIdBytes[:], uint64(lastMilestoneToRemove))
-	for k, _, err := milestoneCursor.Seek(milestoneIdBytes[:]); err == nil && k != nil; k, _, err = milestoneCursor.Prev() {
-		if err = milestoneCursor.DeleteCurrent(); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}
-
 func TruncateCanonicalChain(ctx context.Context, db kv.RwTx, from uint64) error {
 	return db.ForEach(kv.HeaderCanonical, hexutility.EncodeTs(from), func(k, _ []byte) error {
 		return db.Delete(kv.HeaderCanonical, k)