From 6bc97f1ab6ed2e79417eb79d9be4f1e1240e61d5 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 22 May 2024 17:23:04 +0100 Subject: [PATCH 01/38] updated torrent mod to alpha 11 --- erigon-lib/downloader/downloader_grpc_server.go | 9 +++++++++ erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index fb86f5bdc83..4c260ae9ee9 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -21,6 +21,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "time" "github.com/anacrolix/torrent/metainfo" @@ -63,6 +64,14 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque return nil, fmt.Errorf("field 'path' is required") } + const bor_dltest = true + + if bor_dltest { + if !strings.HasSuffix(it.Path, "v1-storage.0-2048.kv") { + continue + } + } + select { case <-logEvery.C: log.Info("[snapshots] initializing", "files", fmt.Sprintf("%d/%d", i, len(request.Items))) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 67fbd276b19..9635fe983c8 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -150,7 +150,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-10 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-11 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 818bf8f5f93..0fc4bf48da3 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJadNkXc= github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= -github.com/erigontech/torrent v1.54.2-alpha-10 h1:MqEorLDG5n2jsNAsSC+TKuZUyExO/KfGumHxh7GHG3o= -github.com/erigontech/torrent v1.54.2-alpha-10/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-11 h1:NZGEDktH4lsITQgKXQwCPd/kigHBGE2PZNWahh0KDt8= +github.com/erigontech/torrent v1.54.2-alpha-11/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index 92b5e6ba672..24ea3a6ec7e 100644 --- a/go.mod +++ b/go.mod @@ -298,6 +298,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-10 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-11 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index fe4ca9b417a..8988b61fcc8 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJad github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-10 h1:MqEorLDG5n2jsNAsSC+TKuZUyExO/KfGumHxh7GHG3o= -github.com/erigontech/torrent v1.54.2-alpha-10/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-11 h1:NZGEDktH4lsITQgKXQwCPd/kigHBGE2PZNWahh0KDt8= +github.com/erigontech/torrent v1.54.2-alpha-11/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From 10bf2b08d02811109508689ccd739c113638690c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 24 May 2024 14:01:21 +0100 Subject: [PATCH 02/38] remove single file test --- erigon-lib/downloader/downloader_grpc_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 4c260ae9ee9..2b6849275ed 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -64,7 +64,7 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque return nil, fmt.Errorf("field 'path' is required") } - const bor_dltest = true + const bor_dltest = false if bor_dltest { if !strings.HasSuffix(it.Path, "v1-storage.0-2048.kv") { From e24f644a637cad2cf7b42f7cb721b86c87fba02b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 24 May 2024 19:07:46 +0100 Subject: [PATCH 03/38] inc from not to --- turbo/snapshotsync/freezeblocks/bor_snapshots.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/turbo/snapshotsync/freezeblocks/bor_snapshots.go b/turbo/snapshotsync/freezeblocks/bor_snapshots.go index 178471742c0..758fb2ea672 100644 --- a/turbo/snapshotsync/freezeblocks/bor_snapshots.go +++ b/turbo/snapshotsync/freezeblocks/bor_snapshots.go @@ -44,7 +44,7 @@ func (br *BlockRetire) retireBorBlocks(ctx context.Context, minBlockNum uint64, continue } - blockFrom, blockTo, ok := canRetire(minSnapNum, maxBlockNum+1, snaptype.Enum(), br.chainConfig) + blockFrom, blockTo, ok := canRetire(minSnapNum+1, maxBlockNum, snaptype.Enum(), br.chainConfig) if ok { blocksRetired = true From 0ee38e70fffc85ee693bf69b8940d265b127a49f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 25 May 2024 12:48:20 +0100 Subject: [PATCH 04/38] updated torrent mod to alpha-12 --- erigon-lib/downloader/downloader_grpc_server.go | 11 +++++++++-- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 2b6849275ed..a58452f0c1d 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -64,13 +64,20 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque return nil, fmt.Errorf("field 'path' is required") } - const bor_dltest = false + // For local Perf Testing + const bor_dltest = "" - if bor_dltest { + switch bor_dltest { + case "last": if !strings.HasSuffix(it.Path, "v1-storage.0-2048.kv") { continue } + case "headers": + if !(strings.Contains(it.Path, "-headers") || strings.Contains(it.Path, "-bodies")) { + continue + } } + // To Here select { case <-logEvery.C: diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 9635fe983c8..2efd2064196 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -150,7 +150,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-11 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-12 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 0fc4bf48da3..d26d138d099 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJadNkXc= github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= -github.com/erigontech/torrent v1.54.2-alpha-11 h1:NZGEDktH4lsITQgKXQwCPd/kigHBGE2PZNWahh0KDt8= -github.com/erigontech/torrent v1.54.2-alpha-11/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-12 h1:od/ai5H4I58AetJ04CJ1lfX1GIG4Y/3BWu3My//3wlc= +github.com/erigontech/torrent v1.54.2-alpha-12/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index 6f71079cf75..0a0babf4033 100644 --- a/go.mod +++ b/go.mod @@ -293,6 +293,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-11 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-12 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index f67e840d537..04017f10eca 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJad github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-11 h1:NZGEDktH4lsITQgKXQwCPd/kigHBGE2PZNWahh0KDt8= -github.com/erigontech/torrent v1.54.2-alpha-11/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-12 h1:od/ai5H4I58AetJ04CJ1lfX1GIG4Y/3BWu3My//3wlc= +github.com/erigontech/torrent v1.54.2-alpha-12/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From 0e238a4e61625a13a03fb4dfe2d96240baeddae4 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 26 May 2024 15:09:19 +0100 Subject: [PATCH 05/38] added active count to roundtrip --- erigon-lib/downloader/downloader.go | 31 ++++++++++++++++++++--------- erigon-lib/go.mod | 4 ++-- erigon-lib/go.sum | 8 ++++---- go.mod | 4 ++-- 4 files changed, 30 insertions(+), 17 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index ab2fc0368dd..862a9ed84f5 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -127,10 +127,12 @@ type AggStats struct { LocalFileHashes int LocalFileHashTime time.Duration - WebseedTripCount *atomic.Int64 - WebseedDiscardCount *atomic.Int64 - WebseedServerFails *atomic.Int64 - WebseedBytesDownload *atomic.Int64 + WebseedActiveTrips *atomic.Int64 + WebseedMaxActiveTrips *atomic.Int64 + WebseedTripCount *atomic.Int64 + WebseedDiscardCount *atomic.Int64 + WebseedServerFails *atomic.Int64 + WebseedBytesDownload *atomic.Int64 lastTorrentStatus time.Time } @@ -177,6 +179,11 @@ func calcBackoff(min, max time.Duration, attemptNum int, resp *http.Response) ti } func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err error) { + activeTrips := r.downloader.stats.WebseedActiveTrips.Add(1) + if activeTrips > r.downloader.stats.WebseedMaxActiveTrips.Load() { + r.downloader.stats.WebseedMaxActiveTrips.Store(activeTrips) + } + defer func() { if r := recover(); r != nil { if resp != nil && resp.Body != nil { @@ -186,6 +193,8 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err err = fmt.Errorf("http client panic: %s", r) } + + r.downloader.stats.WebseedActiveTrips.Add(-1) }() insertCloudflareHeaders(req) @@ -288,10 +297,12 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi mutex := &sync.RWMutex{} stats := AggStats{ - WebseedTripCount: &atomic.Int64{}, - WebseedBytesDownload: &atomic.Int64{}, - WebseedDiscardCount: &atomic.Int64{}, - WebseedServerFails: &atomic.Int64{}, + WebseedActiveTrips: &atomic.Int64{}, + WebseedMaxActiveTrips: &atomic.Int64{}, + WebseedTripCount: &atomic.Int64{}, + WebseedBytesDownload: &atomic.Int64{}, + WebseedDiscardCount: &atomic.Int64{}, + WebseedServerFails: &atomic.Int64{}, } snapLock, err := getSnapshotLock(ctx, cfg, db, &stats, mutex, logger) @@ -2117,13 +2128,15 @@ func (d *Downloader) ReCalcStats(interval time.Duration) { } if !stats.Completed { - logger.Debug("[snapshots] info", + logger.Debug("[snapshots] download info", "len", len(torrents), "webTransfers", webTransfers, "torrent", torrentInfo, "db", dbInfo, "t-complete", tComplete, "webseed-trips", stats.WebseedTripCount.Load(), + "webseed-active", stats.WebseedActiveTrips.Load(), + "webseed-max-active", stats.WebseedMaxActiveTrips.Load(), "webseed-discards", stats.WebseedDiscardCount.Load(), "webseed-fails", stats.WebseedServerFails.Load(), "webseed-bytes", common.ByteCount(uint64(stats.WebseedBytesDownload.Load())), diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 2efd2064196..b3b24955c38 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -56,6 +56,7 @@ require ( github.com/pion/udp v0.1.4 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.20.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect modernc.org/libc v1.50.4 // indirect modernc.org/memory v1.8.0 // indirect modernc.org/sqlite v1.29.8 // indirect @@ -140,9 +141,8 @@ require ( go.opentelemetry.io/otel v1.8.0 // indirect go.opentelemetry.io/otel/trace v1.8.0 // indirect go.uber.org/goleak v1.3.0 // indirect - golang.org/x/net v0.24.0 + golang.org/x/net v0.25.0 golang.org/x/text v0.15.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/mathutil v1.6.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index d26d138d099..e5af842f3f8 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -542,8 +542,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -641,8 +641,8 @@ google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 h1:AgADTJarZTBqgjiUzRgfaBchgYB3/WFTC80GPwsMcRI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291/go.mod h1:EfXuqaE1J41VCDicxHzUDm+8rk+7ZdXzHV0IhO/I6s0= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= diff --git a/go.mod b/go.mod index 0a0babf4033..3f138967b86 100644 --- a/go.mod +++ b/go.mod @@ -117,6 +117,8 @@ require ( github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect ) require ( @@ -276,8 +278,6 @@ require ( golang.org/x/mod v0.17.0 // indirect golang.org/x/text v0.15.0 // indirect golang.org/x/tools v0.21.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gotest.tools/v3 v3.5.1 // indirect From cabd6dc9b864468c43bf45fc76eab7b83cd8d859 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 29 May 2024 13:50:29 +0100 Subject: [PATCH 06/38] change default hashers to numcpu's - 2 --- erigon-lib/downloader/downloadercfg/downloadercfg.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/downloadercfg/downloadercfg.go b/erigon-lib/downloader/downloadercfg/downloadercfg.go index 10f24cabe4d..0b2b5175b10 100644 --- a/erigon-lib/downloader/downloadercfg/downloadercfg.go +++ b/erigon-lib/downloader/downloadercfg/downloadercfg.go @@ -30,6 +30,7 @@ import ( "github.com/anacrolix/torrent" "github.com/c2h5oh/datasize" "github.com/ledgerwatch/erigon-lib/chain/snapcfg" + "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/log/v3" @@ -65,7 +66,7 @@ func Default() *torrent.ClientConfig { // better don't increase because erigon periodically producing "new seedable files" - and adding them to downloader. // it must not impact chain tip sync - so, limit resources to minimum by default. // but when downloader is started as a separated process - rise it to max - //torrentConfig.PieceHashersPerTorrent = cmp.Max(1, runtime.NumCPU()-1) + torrentConfig.PieceHashersPerTorrent = cmp.Max(1, runtime.NumCPU()-2) torrentConfig.MinDialTimeout = 6 * time.Second //default: 3s torrentConfig.HandshakesTimeout = 8 * time.Second //default: 4s From 510be13364fe412ea03189a2569d162310180cb6 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 29 May 2024 19:14:03 +0100 Subject: [PATCH 07/38] force disable peice check --- erigon-lib/downloader/downloader_grpc_server.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index a58452f0c1d..c6f039e7d44 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -65,7 +65,7 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque } // For local Perf Testing - const bor_dltest = "" + const bor_dltest = "headers" switch bor_dltest { case "last": From a880d1ae62d46a81c8b960e7d8def0678dbe42a4 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 9 Jun 2024 09:28:14 +0100 Subject: [PATCH 08/38] post merge fixes --- erigon-lib/downloader/downloader_grpc_server.go | 2 +- erigon-lib/downloader/downloadercfg/downloadercfg.go | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index c6f039e7d44..a58452f0c1d 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -65,7 +65,7 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque } // For local Perf Testing - const bor_dltest = "headers" + const bor_dltest = "" switch bor_dltest { case "last": diff --git a/erigon-lib/downloader/downloadercfg/downloadercfg.go b/erigon-lib/downloader/downloadercfg/downloadercfg.go index 0b2b5175b10..f3c132371ce 100644 --- a/erigon-lib/downloader/downloadercfg/downloadercfg.go +++ b/erigon-lib/downloader/downloadercfg/downloadercfg.go @@ -30,7 +30,6 @@ import ( "github.com/anacrolix/torrent" "github.com/c2h5oh/datasize" "github.com/ledgerwatch/erigon-lib/chain/snapcfg" - "github.com/ledgerwatch/erigon-lib/common/cmp" "github.com/ledgerwatch/erigon-lib/common/datadir" "github.com/ledgerwatch/erigon-lib/common/dir" "github.com/ledgerwatch/log/v3" @@ -66,7 +65,7 @@ func Default() *torrent.ClientConfig { // better don't increase because erigon periodically producing "new seedable files" - and adding them to downloader. // it must not impact chain tip sync - so, limit resources to minimum by default. // but when downloader is started as a separated process - rise it to max - torrentConfig.PieceHashersPerTorrent = cmp.Max(1, runtime.NumCPU()-2) + torrentConfig.PieceHashersPerTorrent = max(1, runtime.NumCPU()-2) torrentConfig.MinDialTimeout = 6 * time.Second //default: 3s torrentConfig.HandshakesTimeout = 8 * time.Second //default: 4s From 1699e507e73c2fdaa6d1f88da0ab680d835980e2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 9 Jun 2024 13:33:36 +0100 Subject: [PATCH 09/38] add additional torrent download if its still unavailable --- erigon-lib/downloader/downloader.go | 13 +++++++++++++ erigon-lib/downloader/downloader_grpc_server.go | 6 +++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 862a9ed84f5..753d0aed769 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -835,6 +835,19 @@ func (d *Downloader) mainLoop(silent bool) error { for _, t := range d.torrentClient.Torrents() { if urls, ok := d.webseeds.ByFileName(t.Name()); ok { + // if we have created a torrent bit it has no info assume that the + // webseed download either has not been called yet or has failed and + // try again here - otherwise the torrent will be left with no info + if t.Info() == nil { + ts, ok, err := d.webseeds.DownloadAndSaveTorrentFile(d.ctx, t.Name()) + if ok && err == nil { + _, _, err = addTorrentFile(d.ctx, ts, d.torrentClient, d.db, d.webseeds) + if err != nil { + continue + } + } + } + t.AddWebSeeds(urls) } } diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index a58452f0c1d..3d9b8157f19 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -65,11 +65,11 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque } // For local Perf Testing - const bor_dltest = "" + const bor_dltest = "storage" switch bor_dltest { - case "last": - if !strings.HasSuffix(it.Path, "v1-storage.0-2048.kv") { + case "storage": + if !strings.Contains(it.Path, "domain/v1-storage") { continue } case "headers": From 14a094093da5b15691bcc1c370fc908f9216c470 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 11 Jun 2024 11:18:53 +0100 Subject: [PATCH 10/38] add completed bitmap processing --- .../downloader/mdbx_piece_completion.go | 94 +++++++++++++++++-- 1 file changed, 87 insertions(+), 7 deletions(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 32e38ea4d2d..b050ae8e0f5 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -19,7 +19,10 @@ package downloader import ( "context" "encoding/binary" + "fmt" + "sync" + "github.com/RoaringBitmap/roaring" "github.com/anacrolix/torrent/metainfo" "github.com/anacrolix/torrent/storage" "github.com/anacrolix/torrent/types/infohash" @@ -33,17 +36,31 @@ const ( ) type mdbxPieceCompletion struct { - db kv.RwDB + db *mdbx.MdbxKV + mu sync.RWMutex + completed map[infohash.T]*roaring.Bitmap } var _ storage.PieceCompletion = (*mdbxPieceCompletion)(nil) func NewMdbxPieceCompletion(db kv.RwDB) (ret storage.PieceCompletion, err error) { - ret = &mdbxPieceCompletion{db: db} + ret = &mdbxPieceCompletion{db: db.(*mdbx.MdbxKV), completed: map[infohash.T]*roaring.Bitmap{}} return } -func (m mdbxPieceCompletion) Get(pk metainfo.PieceKey) (cn storage.Completion, err error) { +func (m *mdbxPieceCompletion) Get(pk metainfo.PieceKey) (cn storage.Completion, err error) { + m.mu.RLock() + if completed, ok := m.completed[pk.InfoHash]; ok { + if completed.Contains(uint32(pk.Index)) { + m.mu.RUnlock() + return storage.Completion{ + Complete: true, + Ok: true, + }, nil + } + } + m.mu.RUnlock() + err = m.db.View(context.Background(), func(tx kv.Tx) error { var key [infohash.Size + 4]byte copy(key[:], pk.InfoHash[:]) @@ -66,11 +83,14 @@ func (m mdbxPieceCompletion) Get(pk metainfo.PieceKey) (cn storage.Completion, e return } -func (m mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { +func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { if c, err := m.Get(pk); err == nil && c.Ok && c.Complete == b { return nil } + m.mu.Lock() + defer m.mu.Unlock() + var tx kv.RwTx var err error // On power-off recent "no-sync" txs may be lost. @@ -84,6 +104,15 @@ func (m mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { // 1K fsyncs/2minutes it's quite expensive, but even on cloud (high latency) drive it allow download 100mb/s // and Erigon doesn't do anything when downloading snapshots if b { + completed, ok := m.completed[pk.InfoHash] + + if !ok { + completed = &roaring.Bitmap{} + m.completed[pk.InfoHash] = completed + } + + completed.Add(uint32(pk.Index)) + tx, err = m.db.BeginRwNosync(context.Background()) if err != nil { return err @@ -112,23 +141,59 @@ func (m mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { return tx.Commit() } +func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitmap) { + m.mu.Lock() + defer m.mu.Unlock() + + if completed, ok := m.completed[infoHash]; ok { + setters := flushed.Clone() + setters.And(completed) + + if setters.GetCardinality() > 0 { + setters.Iterate(func(piece uint32) bool { + //fmt.Println("Set", piece) + return true + }) + } + + completed.AndNot(setters) + + if completed.IsEmpty() { + delete(m.completed, infoHash) + } + } +} + func (m *mdbxPieceCompletion) Close() error { m.db.Close() return nil } type mdbxPieceCompletionBatch struct { - db *mdbx.MdbxKV + mdbxPieceCompletion } var _ storage.PieceCompletion = (*mdbxPieceCompletionBatch)(nil) func NewMdbxPieceCompletionBatch(db kv.RwDB) (ret storage.PieceCompletion, err error) { - ret = &mdbxPieceCompletionBatch{db: db.(*mdbx.MdbxKV)} - return + return &mdbxPieceCompletionBatch{mdbxPieceCompletion{ + db: db.(*mdbx.MdbxKV), + completed: map[infohash.T]*roaring.Bitmap{}}}, nil } func (m *mdbxPieceCompletionBatch) Get(pk metainfo.PieceKey) (cn storage.Completion, err error) { + m.mu.RLock() + if completed, ok := m.completed[pk.InfoHash]; ok { + if completed.Contains(uint32(pk.Index)) { + m.mu.RUnlock() + return storage.Completion{ + Complete: true, + Ok: true, + }, nil + } + } + m.mu.RUnlock() + err = m.db.View(context.Background(), func(tx kv.Tx) error { var key [infohash.Size + 4]byte copy(key[:], pk.InfoHash[:]) @@ -155,10 +220,25 @@ func (m *mdbxPieceCompletionBatch) Set(pk metainfo.PieceKey, b bool) error { if c, err := m.Get(pk); err == nil && c.Ok && c.Complete == b { return nil } + + m.mu.Lock() + defer m.mu.Unlock() + var key [infohash.Size + 4]byte copy(key[:], pk.InfoHash[:]) binary.BigEndian.PutUint32(key[infohash.Size:], uint32(pk.Index)) + if b { + completed, ok := m.completed[pk.InfoHash] + + if !ok { + completed = &roaring.Bitmap{} + m.completed[pk.InfoHash] = completed + } + + completed.Add(uint32(pk.Index)) + } + v := []byte(incomplete) if b { v = []byte(complete) From 7e63f02e4a9eed5cfce2c1f281908e6a0338ba56 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 11 Jun 2024 11:19:23 +0100 Subject: [PATCH 11/38] remove import --- erigon-lib/downloader/mdbx_piece_completion.go | 1 - 1 file changed, 1 deletion(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index b050ae8e0f5..ad2788c1522 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -19,7 +19,6 @@ package downloader import ( "context" "encoding/binary" - "fmt" "sync" "github.com/RoaringBitmap/roaring" From 3ad794cb2e45f5b33b77b5ad10a828d642fe1225 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Tue, 11 Jun 2024 14:42:11 +0100 Subject: [PATCH 12/38] added flush persistence --- .../downloader/mdbx_piece_completion.go | 114 ++++++++---------- 1 file changed, 52 insertions(+), 62 deletions(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index ad2788c1522..7d832fbdab0 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -112,27 +112,18 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { completed.Add(uint32(pk.Index)) - tx, err = m.db.BeginRwNosync(context.Background()) - if err != nil { - return err - } - } else { - tx, err = m.db.BeginRw(context.Background()) - if err != nil { - return err - } + return nil + } + + tx, err = m.db.BeginRw(context.Background()) + if err != nil { + return err } + defer tx.Rollback() - var key [infohash.Size + 4]byte - copy(key[:], pk.InfoHash[:]) - binary.BigEndian.PutUint32(key[infohash.Size:], uint32(pk.Index)) + err = putCompletion(tx, pk.InfoHash, uint32(pk.Index), b) - v := []byte(incomplete) - if b { - v = []byte(complete) - } - err = tx.Put(kv.BittorrentCompletion, key[:], v) if err != nil { return err } @@ -140,17 +131,45 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { return tx.Commit() } +func putCompletion(tx kv.RwTx, infoHash infohash.T, index uint32, c bool) error { + var key [infohash.Size + 4]byte + copy(key[:], infoHash[:]) + binary.BigEndian.PutUint32(key[infohash.Size:], index) + + v := []byte(incomplete) + if c { + v = []byte(complete) + } + //fmt.Println("PUT", infoHash, index, c) + return tx.Put(kv.BittorrentCompletion, key[:], v) +} + func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitmap) { m.mu.Lock() defer m.mu.Unlock() + tx, err := m.db.BeginRwNosync(context.Background()) + + if err != nil { + return + } + + defer tx.Rollback() + + m.putFlushed(tx, infoHash, flushed) + + tx.Commit() +} + +func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { if completed, ok := m.completed[infoHash]; ok { setters := flushed.Clone() setters.And(completed) if setters.GetCardinality() > 0 { setters.Iterate(func(piece uint32) bool { - //fmt.Println("Set", piece) + // TODO deal with error (? don't remove from bitset ?) + _ = putCompletion(tx, infoHash, piece, true) return true }) } @@ -180,41 +199,6 @@ func NewMdbxPieceCompletionBatch(db kv.RwDB) (ret storage.PieceCompletion, err e completed: map[infohash.T]*roaring.Bitmap{}}}, nil } -func (m *mdbxPieceCompletionBatch) Get(pk metainfo.PieceKey) (cn storage.Completion, err error) { - m.mu.RLock() - if completed, ok := m.completed[pk.InfoHash]; ok { - if completed.Contains(uint32(pk.Index)) { - m.mu.RUnlock() - return storage.Completion{ - Complete: true, - Ok: true, - }, nil - } - } - m.mu.RUnlock() - - err = m.db.View(context.Background(), func(tx kv.Tx) error { - var key [infohash.Size + 4]byte - copy(key[:], pk.InfoHash[:]) - binary.BigEndian.PutUint32(key[infohash.Size:], uint32(pk.Index)) - cn.Ok = true - v, err := tx.GetOne(kv.BittorrentCompletion, key[:]) - if err != nil { - return err - } - switch string(v) { - case complete: - cn.Complete = true - case incomplete: - cn.Complete = false - default: - cn.Ok = false - } - return nil - }) - return -} - func (m *mdbxPieceCompletionBatch) Set(pk metainfo.PieceKey, b bool) error { if c, err := m.Get(pk); err == nil && c.Ok && c.Complete == b { return nil @@ -223,10 +207,6 @@ func (m *mdbxPieceCompletionBatch) Set(pk metainfo.PieceKey, b bool) error { m.mu.Lock() defer m.mu.Unlock() - var key [infohash.Size + 4]byte - copy(key[:], pk.InfoHash[:]) - binary.BigEndian.PutUint32(key[infohash.Size:], uint32(pk.Index)) - if b { completed, ok := m.completed[pk.InfoHash] @@ -236,18 +216,28 @@ func (m *mdbxPieceCompletionBatch) Set(pk metainfo.PieceKey, b bool) error { } completed.Add(uint32(pk.Index)) - } - v := []byte(incomplete) - if b { - v = []byte(complete) + return nil } + return m.db.Batch(func(tx kv.RwTx) error { - return tx.Put(kv.BittorrentCompletion, key[:], v) + return putCompletion(tx, pk.InfoHash, uint32(pk.Index), b) + }) +} + +func (m *mdbxPieceCompletionBatch) Flushed(infoHash infohash.T, flushed *roaring.Bitmap) { + m.mu.Lock() + defer m.mu.Unlock() + //fmt.Println("FLUSH", infoHash) + m.db.Batch(func(tx kv.RwTx) error { + m.putFlushed(tx, infoHash, flushed) + return nil }) } func (m *mdbxPieceCompletionBatch) Close() error { + m.mu.Lock() + defer m.mu.Unlock() m.db.Close() return nil } From ada1a49febb94830cb1de6ca64f243a350cf03ad Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 12 Jun 2024 15:54:26 +0100 Subject: [PATCH 13/38] remove temp test code --- erigon-lib/downloader/downloader_grpc_server.go | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/erigon-lib/downloader/downloader_grpc_server.go b/erigon-lib/downloader/downloader_grpc_server.go index 3d9b8157f19..fb86f5bdc83 100644 --- a/erigon-lib/downloader/downloader_grpc_server.go +++ b/erigon-lib/downloader/downloader_grpc_server.go @@ -21,7 +21,6 @@ import ( "fmt" "os" "path/filepath" - "strings" "time" "github.com/anacrolix/torrent/metainfo" @@ -64,21 +63,6 @@ func (s *GrpcServer) Add(ctx context.Context, request *proto_downloader.AddReque return nil, fmt.Errorf("field 'path' is required") } - // For local Perf Testing - const bor_dltest = "storage" - - switch bor_dltest { - case "storage": - if !strings.Contains(it.Path, "domain/v1-storage") { - continue - } - case "headers": - if !(strings.Contains(it.Path, "-headers") || strings.Contains(it.Path, "-bodies")) { - continue - } - } - // To Here - select { case <-logEvery.C: log.Info("[snapshots] initializing", "files", fmt.Sprintf("%d/%d", i, len(request.Items))) From c0757ac9c320a1b67e9855601235fc9ad5ecc521 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 12 Jun 2024 17:41:32 +0100 Subject: [PATCH 14/38] updated mods for erigontech/torrent release --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 12 +++++++----- go.sum | 16 ++++++++-------- 4 files changed, 18 insertions(+), 16 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index d8d71d4b4a5..fdd725f004d 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -150,7 +150,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-12 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-15 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index ae0b22f5fe4..eb3487a7da2 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJadNkXc= github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= -github.com/erigontech/torrent v1.54.2-alpha-12 h1:od/ai5H4I58AetJ04CJ1lfX1GIG4Y/3BWu3My//3wlc= -github.com/erigontech/torrent v1.54.2-alpha-12/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-15 h1:Mrax7PZ4MY+1qnpnaUnK1nFnEuPv+ouNLMF3Ijehi50= +github.com/erigontech/torrent v1.54.2-alpha-15/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index e0400873f86..a69a31d433b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/ledgerwatch/erigon -go 1.21.5 +go 1.22 + +toolchain go1.22.2 require ( github.com/erigontech/mdbx-go v0.38.0 @@ -129,12 +131,12 @@ require ( github.com/anacrolix/chansync v0.3.0 // indirect github.com/anacrolix/dht/v2 v2.21.1 // indirect github.com/anacrolix/envpprof v1.3.0 // indirect - github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 // indirect + github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab // indirect github.com/anacrolix/go-libutp v1.3.1 // indirect github.com/anacrolix/log v0.15.2 // indirect github.com/anacrolix/missinggo v1.3.0 // indirect github.com/anacrolix/missinggo/perf v1.0.0 // indirect - github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 // indirect + github.com/anacrolix/missinggo/v2 v2.7.3 // indirect github.com/anacrolix/mmsg v1.0.0 // indirect github.com/anacrolix/multiless v0.3.1-0.20221221005021-2d12701f83f7 // indirect github.com/anacrolix/stm v0.4.1-0.20221221005312-96d17df0e496 // indirect @@ -167,7 +169,7 @@ require ( github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/garslo/gogen v0.0.0-20170307003452-d6ebae628c7c // indirect github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 // indirect - github.com/go-llsqlite/crawshaw v0.4.0 // indirect + github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 // indirect github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect @@ -293,6 +295,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-12 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-15 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 374c185322f..c939c48e523 100644 --- a/go.sum +++ b/go.sum @@ -103,8 +103,8 @@ github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54g github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= github.com/anacrolix/envpprof v1.3.0 h1:WJt9bpuT7A/CDCxPOv/eeZqHWlle/Y0keJUvc6tcJDk= github.com/anacrolix/envpprof v1.3.0/go.mod h1:7QIG4CaX1uexQ3tqd5+BRa/9e2D02Wcertl6Yh0jCB0= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45 h1:Kmcl3I9K2+5AdnnR7hvrnVT0TLeFWWMa9bxnm55aVIg= -github.com/anacrolix/generics v0.0.0-20230816105729-c755655aee45/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab h1:MvuAC/UJtcohN6xWc8zYXSZfllh1LVNepQ0R3BCX5I4= +github.com/anacrolix/generics v0.0.2-0.20240227122613-f95486179cab/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= github.com/anacrolix/go-libutp v1.3.1 h1:idJzreNLl+hNjGC3ZnUOjujEaryeOGgkwHLqSGoige0= github.com/anacrolix/go-libutp v1.3.1/go.mod h1:heF41EC8kN0qCLMokLBVkB8NXiLwx3t8R8810MTNI5o= github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= @@ -126,8 +126,8 @@ github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5ur github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9 h1:W/oGeHhYwxueeiDjQfmK9G+X9M2xJgfTtow62v0TWAs= -github.com/anacrolix/missinggo/v2 v2.7.2-0.20230527121029-a582b4f397b9/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= +github.com/anacrolix/missinggo/v2 v2.7.3 h1:Ee//CmZBMadeNiYB/hHo9ly2PFOEZ4Fhsbnug3rDAIE= +github.com/anacrolix/missinggo/v2 v2.7.3/go.mod h1:mIEtp9pgaXqt8VQ3NQxFOod/eQ1H0D1XsZzKUQfwtac= github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw= github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJad github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-12 h1:od/ai5H4I58AetJ04CJ1lfX1GIG4Y/3BWu3My//3wlc= -github.com/erigontech/torrent v1.54.2-alpha-12/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-15 h1:Mrax7PZ4MY+1qnpnaUnK1nFnEuPv+ouNLMF3Ijehi50= +github.com/erigontech/torrent v1.54.2-alpha-15/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -323,8 +323,8 @@ github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2 github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916 h1:OyQmpAN302wAopDgwVjgs2HkFawP9ahIEqkUYz7V7CA= github.com/go-llsqlite/adapter v0.0.0-20230927005056-7f5ce7f0c916/go.mod h1:DADrR88ONKPPeSGjFp5iEN55Arx3fi2qXZeKCYDpbmU= -github.com/go-llsqlite/crawshaw v0.4.0 h1:L02s2jZBBJj80xm1VkkdyB/JlQ/Fi0kLbNHfXA8yrec= -github.com/go-llsqlite/crawshaw v0.4.0/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568 h1:3EpZo8LxIzF4q3BT+vttQQlRfA6uTtTb/cxVisWa5HM= +github.com/go-llsqlite/crawshaw v0.5.2-0.20240425034140-f30eb7704568/go.mod h1:/YJdV7uBQaYDE0fwe4z3wwJIZBJxdYzd38ICggWqtaE= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= From 62452c0fe9062901589e21ef904ee73189174bb3 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 13 Jun 2024 18:44:43 +0100 Subject: [PATCH 15/38] update mods --- erigon-lib/go.mod | 4 +++- erigon-lib/go.sum | 8 ++++++-- go.mod | 4 +++- go.sum | 8 ++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 91bcbe5900a..6c65288ed73 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -53,7 +53,9 @@ require ( github.com/cespare/xxhash v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/udp v0.1.4 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.20.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect @@ -150,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-15 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-16 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 7703827884a..6abcdd76b40 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJadNkXc= github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= -github.com/erigontech/torrent v1.54.2-alpha-15 h1:Mrax7PZ4MY+1qnpnaUnK1nFnEuPv+ouNLMF3Ijehi50= -github.com/erigontech/torrent v1.54.2-alpha-15/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= +github.com/erigontech/torrent v1.54.2-alpha-16 h1:kL/UTzv0420gndsRAS+xGSRkkWMRPXDJHvFB0QNp8jk= +github.com/erigontech/torrent v1.54.2-alpha-16/go.mod h1:HfUQBVyfhEjECBU0K67vPwoKsGZuUcA3NsH/UqaLR6g= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -327,6 +327,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= @@ -419,6 +421,8 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= diff --git a/go.mod b/go.mod index 68f6aa035e4..e8b05ecb583 100644 --- a/go.mod +++ b/go.mod @@ -116,6 +116,8 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.6 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -295,6 +297,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-15 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-16 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 80c46b6dffc..89be8a49d2a 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.0 h1:K64h6YHc2biN081DPEp/KP1TE+X0Jmxu8T+RJad github.com/erigontech/mdbx-go v0.38.0/go.mod h1:FAMxbOgqOnRDx51j8HjuJZIgznbDwjX7LItd+/UWyA4= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-15 h1:Mrax7PZ4MY+1qnpnaUnK1nFnEuPv+ouNLMF3Ijehi50= -github.com/erigontech/torrent v1.54.2-alpha-15/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= +github.com/erigontech/torrent v1.54.2-alpha-16 h1:kL/UTzv0420gndsRAS+xGSRkkWMRPXDJHvFB0QNp8jk= +github.com/erigontech/torrent v1.54.2-alpha-16/go.mod h1:HfUQBVyfhEjECBU0K67vPwoKsGZuUcA3NsH/UqaLR6g= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -683,6 +683,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= @@ -806,6 +808,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= From 0155fa21b1311a551563fbec8cd215f251ff7c08 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Thu, 13 Jun 2024 18:49:23 +0100 Subject: [PATCH 16/38] tidied mods --- erigon-lib/go.mod | 3 ++- erigon-lib/go.sum | 12 ++++++------ go.mod | 2 -- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 5f69c9913eb..f21f2821b05 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -59,6 +59,8 @@ require ( github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/udp v0.1.4 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect + golang.org/x/mod v0.17.0 // indirect + golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect modernc.org/libc v1.50.4 // indirect modernc.org/memory v1.8.0 // indirect @@ -145,7 +147,6 @@ require ( go.uber.org/goleak v1.3.0 // indirect golang.org/x/net v0.26.0 golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/mathutil v1.6.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 0c27ba0ac10..28ec5582e14 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -148,8 +148,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.1-0.20240612085237-ca514b2384a4 h1:PjUMKTFjT5aBZ3/jKK/ReCQ5AzWtaQcbMBTrVj3Kdcg= github.com/erigontech/mdbx-go v0.38.1-0.20240612085237-ca514b2384a4/go.mod h1:Ndex795deaHs76GzK0hO0IfmZHkMtz6FBiJ2W+KirMw= -github.com/erigontech/torrent v1.54.2-alpha-10 h1:MqEorLDG5n2jsNAsSC+TKuZUyExO/KfGumHxh7GHG3o= -github.com/erigontech/torrent v1.54.2-alpha-10/go.mod h1:nYNeuR4xPlEl4CturFD9/KRXBRJEcJGqjegDNWakwG4= +github.com/erigontech/torrent v1.54.2-alpha-16 h1:kL/UTzv0420gndsRAS+xGSRkkWMRPXDJHvFB0QNp8jk= +github.com/erigontech/torrent v1.54.2-alpha-16/go.mod h1:HfUQBVyfhEjECBU0K67vPwoKsGZuUcA3NsH/UqaLR6g= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -520,8 +520,8 @@ golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.18.0 h1:5+9lSbEzPSdWkH32vYPBwEpX8KwDbM52Ud9xBUvNlb0= -golang.org/x/mod v0.18.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -635,8 +635,8 @@ golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roY golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.22.0 h1:gqSGLZqv+AI9lIQzniJ0nZDRG5GBPsSi+DRNHWNz6yA= -golang.org/x/tools v0.22.0/go.mod h1:aCwcsjqvq7Yqt6TNyX7QMU2enbQ/Gt0bo6krSeEri+c= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/go.mod b/go.mod index 576c74f9f29..2712d394c65 100644 --- a/go.mod +++ b/go.mod @@ -282,8 +282,6 @@ require ( golang.org/x/mod v0.18.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/tools v0.22.0 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20240515191416-fc5f0ca64291 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect gotest.tools/v3 v3.5.1 // indirect From 997cb9dab25875166b5a40b78fb5f534c6eba4c2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 17 Jun 2024 10:26:24 +0700 Subject: [PATCH 17/38] save --- erigon-lib/downloader/downloadercfg/downloadercfg.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/downloadercfg/downloadercfg.go b/erigon-lib/downloader/downloadercfg/downloadercfg.go index f3c132371ce..0d905de49c1 100644 --- a/erigon-lib/downloader/downloadercfg/downloadercfg.go +++ b/erigon-lib/downloader/downloadercfg/downloadercfg.go @@ -17,6 +17,7 @@ package downloadercfg import ( + "github.com/ledgerwatch/erigon-lib/common/dbg" "net" "net/url" "os" @@ -65,7 +66,7 @@ func Default() *torrent.ClientConfig { // better don't increase because erigon periodically producing "new seedable files" - and adding them to downloader. // it must not impact chain tip sync - so, limit resources to minimum by default. // but when downloader is started as a separated process - rise it to max - torrentConfig.PieceHashersPerTorrent = max(1, runtime.NumCPU()-2) + torrentConfig.PieceHashersPerTorrent = dbg.EnvInt("DL_HASHERS", min(16, max(2, runtime.NumCPU()-2))) torrentConfig.MinDialTimeout = 6 * time.Second //default: 3s torrentConfig.HandshakesTimeout = 8 * time.Second //default: 4s From e3011352dfeaf493a792582adc7bec2f7b2a64d2 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 22 Jun 2024 18:38:05 +0100 Subject: [PATCH 18/38] copy stats atomics to avoid race reports --- erigon-lib/downloader/downloader.go | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index e059627c038..d45413e6605 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -179,9 +179,18 @@ func calcBackoff(min, max time.Duration, attemptNum int, resp *http.Response) ti } func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err error) { - activeTrips := r.downloader.stats.WebseedActiveTrips.Add(1) - if activeTrips > r.downloader.stats.WebseedMaxActiveTrips.Load() { - r.downloader.stats.WebseedMaxActiveTrips.Store(activeTrips) + r.downloader.lock.RLock() + webseedMaxActiveTrips := r.downloader.stats.WebseedMaxActiveTrips + webseedActiveTrips := r.downloader.stats.WebseedActiveTrips + webseedTripCount := r.downloader.stats.WebseedTripCount + webseedBytesDownload := r.downloader.stats.WebseedBytesDownload + webseedDiscardCount := r.downloader.stats.WebseedDiscardCount + WebseedServerFails := r.downloader.stats.WebseedServerFails + r.downloader.lock.RUnlock() + + activeTrips := webseedActiveTrips.Add(1) + if activeTrips > webseedMaxActiveTrips.Load() { + webseedMaxActiveTrips.Store(activeTrips) } defer func() { @@ -194,7 +203,7 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err err = fmt.Errorf("http client panic: %s", r) } - r.downloader.stats.WebseedActiveTrips.Add(-1) + webseedActiveTrips.Add(-1) }() insertCloudflareHeaders(req) @@ -209,7 +218,7 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err const maxAttempts = 10 for err == nil && retry { - r.downloader.stats.WebseedTripCount.Add(1) + webseedTripCount.Add(1) switch resp.StatusCode { case http.StatusOK: @@ -222,10 +231,10 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err // TODO: We could count the bytes - probably need to take this from the req though // as its not clear the amount of the content which will be read. This needs // further investigation - if required. - r.downloader.stats.WebseedDiscardCount.Add(1) + webseedDiscardCount.Add(1) } - r.downloader.stats.WebseedBytesDownload.Add(resp.ContentLength) + webseedBytesDownload.Add(resp.ContentLength) retry = false // the first two statuses here have been observed from cloudflare @@ -237,7 +246,7 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err http.StatusTooManyRequests, http.StatusServiceUnavailable, http.StatusGatewayTimeout: - r.downloader.stats.WebseedServerFails.Add(1) + WebseedServerFails.Add(1) if resp.Body != nil { resp.Body.Close() @@ -251,7 +260,7 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err case <-delayTimer.C: // Note this assumes the req.Body is nil resp, err = r.Transport.RoundTrip(req) - r.downloader.stats.WebseedTripCount.Add(1) + webseedTripCount.Add(1) case <-req.Context().Done(): err = req.Context().Err() @@ -259,7 +268,7 @@ func (r *requestHandler) RoundTrip(req *http.Request) (resp *http.Response, err retry = attempts < maxAttempts default: - r.downloader.stats.WebseedBytesDownload.Add(resp.ContentLength) + webseedBytesDownload.Add(resp.ContentLength) retry = false } } From 1d983b3b314224956bc8dcfc0e4557ac5306a501 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 23 Jun 2024 14:56:08 +0100 Subject: [PATCH 19/38] remove db writer race --- erigon-lib/diagnostics/stages.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/erigon-lib/diagnostics/stages.go b/erigon-lib/diagnostics/stages.go index 41115768874..7d1f40814ca 100644 --- a/erigon-lib/diagnostics/stages.go +++ b/erigon-lib/diagnostics/stages.go @@ -101,11 +101,12 @@ func (d *DiagnosticClient) runSyncStagesListListener(rootCtx context.Context) { case <-rootCtx.Done(): return case info := <-ch: - d.mu.Lock() - d.SetStagesList(info.StagesList) - d.mu.Unlock() - - d.saveSyncStagesToDB() + func() { + d.mu.Lock() + defer d.mu.Unlock() + d.SetStagesList(info.StagesList) + d.saveSyncStagesToDB() + }() } } }() @@ -233,6 +234,8 @@ func (d *DiagnosticClient) SetCurrentSyncStage(css CurrentSyncStage) error { } isSet := false + d.mu.Lock() + defer d.mu.Unlock() for idx, stage := range d.syncStages { if !isSet { if stage.ID == css.Stage { From 67a41c9755d505ecf2d2be20d9aa1f19b2d61c9a Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 23 Jun 2024 17:54:55 +0100 Subject: [PATCH 20/38] fix race reading stats --- erigon-lib/downloader/downloader.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index d45413e6605..894de92f642 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -1089,10 +1089,10 @@ func (d *Downloader) mainLoop(silent bool) error { } } - d.lock.RLock() + d.lock.Lock() downloadingLen := len(d.downloading) d.stats.Downloading = int32(downloadingLen) - d.lock.RUnlock() + d.lock.Unlock() // the call interval of the loop (elapsed sec) used to get slots/sec for // calculating the number of files to download based on the loop speed From 952d0d34fb5be3356f1d50f1676f90c12bef5d3f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 24 Jun 2024 20:01:41 +0100 Subject: [PATCH 21/38] update mods to latest torrent lib --- erigon-lib/go.mod | 4 +--- erigon-lib/go.sum | 8 ++------ go.mod | 4 +--- go.sum | 8 ++------ 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index d4e941a8c18..978f09cf1a8 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -54,9 +54,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/udp v0.1.4 // indirect - github.com/sasha-s/go-deadlock v0.3.1 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect @@ -152,7 +150,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-16 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-18 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 98d71771450..2008427442f 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-16 h1:kL/UTzv0420gndsRAS+xGSRkkWMRPXDJHvFB0QNp8jk= -github.com/erigontech/torrent v1.54.2-alpha-16/go.mod h1:HfUQBVyfhEjECBU0K67vPwoKsGZuUcA3NsH/UqaLR6g= +github.com/erigontech/torrent v1.54.2-alpha-18 h1:9W+NAJQ2aJZDbx3huOEpqreLBUMQ9quz2UXhlQWu2X8= +github.com/erigontech/torrent v1.54.2-alpha-18/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -326,8 +326,6 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= @@ -420,8 +418,6 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= -github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= diff --git a/go.mod b/go.mod index 9846a5b7d03..63adb485c8a 100644 --- a/go.mod +++ b/go.mod @@ -115,8 +115,6 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.6 // indirect - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect - github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -296,6 +294,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-16 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-18 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 7017fbb43e6..d4a896910e8 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-16 h1:kL/UTzv0420gndsRAS+xGSRkkWMRPXDJHvFB0QNp8jk= -github.com/erigontech/torrent v1.54.2-alpha-16/go.mod h1:HfUQBVyfhEjECBU0K67vPwoKsGZuUcA3NsH/UqaLR6g= +github.com/erigontech/torrent v1.54.2-alpha-18 h1:9W+NAJQ2aJZDbx3huOEpqreLBUMQ9quz2UXhlQWu2X8= +github.com/erigontech/torrent v1.54.2-alpha-18/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -680,8 +680,6 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= @@ -805,8 +803,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= -github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= From e1fa6a39091820f32c3647b3c861f103b636ae2d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 25 Jun 2024 16:40:08 +0700 Subject: [PATCH 22/38] fix deadlock after merge main --- erigon-lib/diagnostics/stages.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/erigon-lib/diagnostics/stages.go b/erigon-lib/diagnostics/stages.go index 616a515ea53..6cbe6586caa 100644 --- a/erigon-lib/diagnostics/stages.go +++ b/erigon-lib/diagnostics/stages.go @@ -235,8 +235,6 @@ func (d *DiagnosticClient) SetCurrentSyncStage(css CurrentSyncStage) error { } isSet := false - d.mu.Lock() - defer d.mu.Unlock() for idx, stage := range d.syncStages { if !isSet { if stage.ID == css.Stage { From b888d165e7b37acfd49df609666f4dcbfb8cd923 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Wed, 26 Jun 2024 21:31:52 +0100 Subject: [PATCH 23/38] updated piece completion processing --- erigon-lib/downloader/downloader.go | 7 +- .../downloader/mdbx_piece_completion.go | 73 ++++--------------- .../downloader/mdbx_piece_completion_test.go | 28 +------ 3 files changed, 20 insertions(+), 88 deletions(-) diff --git a/erigon-lib/downloader/downloader.go b/erigon-lib/downloader/downloader.go index 6c0d3e05557..5365517b5c0 100644 --- a/erigon-lib/downloader/downloader.go +++ b/erigon-lib/downloader/downloader.go @@ -288,7 +288,7 @@ func New(ctx context.Context, cfg *downloadercfg.Cfg, logger log.Logger, verbosi cfg.ClientConfig.WebTransport = requestHandler - db, c, m, torrentClient, err := openClient(ctx, cfg.Dirs.Downloader, cfg.Dirs.Snap, cfg.ClientConfig, cfg.MdbxWriteMap) + db, c, m, torrentClient, err := openClient(ctx, cfg.Dirs.Downloader, cfg.Dirs.Snap, cfg.ClientConfig, cfg.MdbxWriteMap, logger) if err != nil { return nil, fmt.Errorf("openClient: %w", err) } @@ -835,6 +835,7 @@ func (d *Downloader) mainLoop(silent bool) error { if ok && err == nil { _, _, err = addTorrentFile(d.ctx, ts, d.torrentClient, d.db, d.webseeds) if err != nil { + d.logger.Warn("[snapshots] addTorrentFile from webseed", "err", err) continue } } @@ -2702,7 +2703,7 @@ func (d *Downloader) StopSeeding(hash metainfo.Hash) error { func (d *Downloader) TorrentClient() *torrent.Client { return d.torrentClient } -func openClient(ctx context.Context, dbDir, snapDir string, cfg *torrent.ClientConfig, writeMap bool) (db kv.RwDB, c storage.PieceCompletion, m storage.ClientImplCloser, torrentClient *torrent.Client, err error) { +func openClient(ctx context.Context, dbDir, snapDir string, cfg *torrent.ClientConfig, writeMap bool, logger log.Logger) (db kv.RwDB, c storage.PieceCompletion, m storage.ClientImplCloser, torrentClient *torrent.Client, err error) { dbCfg := mdbx.NewMDBX(log.New()). Label(kv.DownloaderDB). WithTableCfg(func(defaultBuckets kv.TableCfg) kv.TableCfg { return kv.DownloaderTablesCfg }). @@ -2720,7 +2721,7 @@ func openClient(ctx context.Context, dbDir, snapDir string, cfg *torrent.ClientC return nil, nil, nil, nil, fmt.Errorf("torrentcfg.openClient: %w", err) } //c, err = NewMdbxPieceCompletion(db) - c, err = NewMdbxPieceCompletionBatch(db) + c, err = NewMdbxPieceCompletion(db, logger) if err != nil { return nil, nil, nil, nil, fmt.Errorf("torrentcfg.NewMdbxPieceCompletion: %w", err) } diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 7d832fbdab0..2aef159ba3f 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -27,6 +27,7 @@ import ( "github.com/anacrolix/torrent/types/infohash" "github.com/ledgerwatch/erigon-lib/kv" "github.com/ledgerwatch/erigon-lib/kv/mdbx" + "github.com/ledgerwatch/erigon-lib/log/v3" ) const ( @@ -38,12 +39,16 @@ type mdbxPieceCompletion struct { db *mdbx.MdbxKV mu sync.RWMutex completed map[infohash.T]*roaring.Bitmap + logger log.Logger } var _ storage.PieceCompletion = (*mdbxPieceCompletion)(nil) -func NewMdbxPieceCompletion(db kv.RwDB) (ret storage.PieceCompletion, err error) { - ret = &mdbxPieceCompletion{db: db.(*mdbx.MdbxKV), completed: map[infohash.T]*roaring.Bitmap{}} +func NewMdbxPieceCompletion(db kv.RwDB, logger log.Logger) (ret storage.PieceCompletion, err error) { + ret = &mdbxPieceCompletion{ + db: db.(*mdbx.MdbxKV), + logger: logger, + completed: map[infohash.T]*roaring.Bitmap{}} return } @@ -148,9 +153,10 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm m.mu.Lock() defer m.mu.Unlock() - tx, err := m.db.BeginRwNosync(context.Background()) + tx, err := m.db.BeginRw(context.Background()) if err != nil { + m.logger.Warn("[snapshots] failed to flush piece completions", "hash", infoHash, err, err) return } @@ -158,7 +164,11 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm m.putFlushed(tx, infoHash, flushed) - tx.Commit() + err = tx.Commit() + + if err != nil { + m.logger.Warn("[snapshots] failed to flush piece completions", "hash", infoHash, err, err) + } } func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { @@ -186,58 +196,3 @@ func (m *mdbxPieceCompletion) Close() error { m.db.Close() return nil } - -type mdbxPieceCompletionBatch struct { - mdbxPieceCompletion -} - -var _ storage.PieceCompletion = (*mdbxPieceCompletionBatch)(nil) - -func NewMdbxPieceCompletionBatch(db kv.RwDB) (ret storage.PieceCompletion, err error) { - return &mdbxPieceCompletionBatch{mdbxPieceCompletion{ - db: db.(*mdbx.MdbxKV), - completed: map[infohash.T]*roaring.Bitmap{}}}, nil -} - -func (m *mdbxPieceCompletionBatch) Set(pk metainfo.PieceKey, b bool) error { - if c, err := m.Get(pk); err == nil && c.Ok && c.Complete == b { - return nil - } - - m.mu.Lock() - defer m.mu.Unlock() - - if b { - completed, ok := m.completed[pk.InfoHash] - - if !ok { - completed = &roaring.Bitmap{} - m.completed[pk.InfoHash] = completed - } - - completed.Add(uint32(pk.Index)) - - return nil - } - - return m.db.Batch(func(tx kv.RwTx) error { - return putCompletion(tx, pk.InfoHash, uint32(pk.Index), b) - }) -} - -func (m *mdbxPieceCompletionBatch) Flushed(infoHash infohash.T, flushed *roaring.Bitmap) { - m.mu.Lock() - defer m.mu.Unlock() - //fmt.Println("FLUSH", infoHash) - m.db.Batch(func(tx kv.RwTx) error { - m.putFlushed(tx, infoHash, flushed) - return nil - }) -} - -func (m *mdbxPieceCompletionBatch) Close() error { - m.mu.Lock() - defer m.mu.Unlock() - m.db.Close() - return nil -} diff --git a/erigon-lib/downloader/mdbx_piece_completion_test.go b/erigon-lib/downloader/mdbx_piece_completion_test.go index a594e3dd897..ee5fca0f783 100644 --- a/erigon-lib/downloader/mdbx_piece_completion_test.go +++ b/erigon-lib/downloader/mdbx_piece_completion_test.go @@ -20,6 +20,7 @@ import ( "testing" "github.com/ledgerwatch/erigon-lib/kv/memdb" + "github.com/ledgerwatch/erigon-lib/log/v3" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -29,32 +30,7 @@ import ( func TestMdbxPieceCompletion(t *testing.T) { db := memdb.NewTestDownloaderDB(t) - pc, err := NewMdbxPieceCompletion(db) - require.NoError(t, err) - defer pc.Close() - - pk := metainfo.PieceKey{} - - b, err := pc.Get(pk) - require.NoError(t, err) - assert.False(t, b.Ok) - - require.NoError(t, pc.Set(pk, false)) - - b, err = pc.Get(pk) - require.NoError(t, err) - assert.Equal(t, storage.Completion{Complete: false, Ok: true}, b) - - require.NoError(t, pc.Set(pk, true)) - - b, err = pc.Get(pk) - require.NoError(t, err) - assert.Equal(t, storage.Completion{Complete: true, Ok: true}, b) -} - -func TestMdbxPieceCompletionBatch(t *testing.T) { - db := memdb.NewTestDownloaderDB(t) - pc, err := NewMdbxPieceCompletionBatch(db) + pc, err := NewMdbxPieceCompletion(db, log.New()) require.NoError(t, err) defer pc.Close() From 08f697b168483ead547f2af5f3e9dab14f68777f Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Fri, 28 Jun 2024 11:23:35 +0100 Subject: [PATCH 24/38] updated torrent with stable memory pool --- erigon-lib/go.mod | 4 +++- erigon-lib/go.sum | 8 ++++++-- go.mod | 4 +++- go.sum | 8 ++++++-- 4 files changed, 18 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 3bd2c07a806..eb12b4ee695 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -54,7 +54,9 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/udp v0.1.4 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect @@ -150,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-18 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-19 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index b11bceeb388..63c933278a7 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-18 h1:9W+NAJQ2aJZDbx3huOEpqreLBUMQ9quz2UXhlQWu2X8= -github.com/erigontech/torrent v1.54.2-alpha-18/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-19 h1:dkl97UlgJXkcX0939q6pWYzadHqHrJ+aBf3xP/GcORk= +github.com/erigontech/torrent v1.54.2-alpha-19/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -326,6 +326,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= @@ -418,6 +420,8 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= diff --git a/go.mod b/go.mod index 0b8a84575a1..ee28ed04a01 100644 --- a/go.mod +++ b/go.mod @@ -115,6 +115,8 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.6 // indirect + github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect + github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -294,6 +296,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-18 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-19 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index d542f9104d8..6aa60c015db 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-18 h1:9W+NAJQ2aJZDbx3huOEpqreLBUMQ9quz2UXhlQWu2X8= -github.com/erigontech/torrent v1.54.2-alpha-18/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-19 h1:dkl97UlgJXkcX0939q6pWYzadHqHrJ+aBf3xP/GcORk= +github.com/erigontech/torrent v1.54.2-alpha-19/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -680,6 +680,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= +github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= @@ -803,6 +805,8 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= +github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= +github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= From 9b9adbe03f6dd40d1df3127e1acb10cba9bb7e1c Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 29 Jun 2024 08:58:53 +0100 Subject: [PATCH 25/38] updated erigontech/torrent to v1.54.2-alpha-20 --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index eb12b4ee695..73d33aea8f8 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -152,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-19 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-20 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 63c933278a7..e66812de302 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-19 h1:dkl97UlgJXkcX0939q6pWYzadHqHrJ+aBf3xP/GcORk= -github.com/erigontech/torrent v1.54.2-alpha-19/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-20 h1:rEcW6hR0+pgCsTXnP8EL7O9o2Oyg3kp5eFGP91nKLyA= +github.com/erigontech/torrent v1.54.2-alpha-20/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index ee28ed04a01..ad61da885ea 100644 --- a/go.mod +++ b/go.mod @@ -296,6 +296,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-19 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-20 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 6aa60c015db..05d10bb8a95 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-19 h1:dkl97UlgJXkcX0939q6pWYzadHqHrJ+aBf3xP/GcORk= -github.com/erigontech/torrent v1.54.2-alpha-19/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-20 h1:rEcW6hR0+pgCsTXnP8EL7O9o2Oyg3kp5eFGP91nKLyA= +github.com/erigontech/torrent v1.54.2-alpha-20/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From 61e667ef2972022e2d738a8587919904c6909379 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 29 Jun 2024 19:55:42 +0100 Subject: [PATCH 26/38] updated erigontech/torrent to v1.54.2-alpha-21 --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 313ebbcc77f..3414c15a489 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -152,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-20 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-21 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 7df77827c99..7ece996ac2b 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-20 h1:rEcW6hR0+pgCsTXnP8EL7O9o2Oyg3kp5eFGP91nKLyA= -github.com/erigontech/torrent v1.54.2-alpha-20/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-21 h1:ig8p6cGK8p34YL1itx+8dXwcEpahcjRQMttOgaJH6Nw= +github.com/erigontech/torrent v1.54.2-alpha-21/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index ab51bd57b05..0fadd93dac6 100644 --- a/go.mod +++ b/go.mod @@ -296,6 +296,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-20 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-21 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index f34462c7e08..f663c51aba7 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-20 h1:rEcW6hR0+pgCsTXnP8EL7O9o2Oyg3kp5eFGP91nKLyA= -github.com/erigontech/torrent v1.54.2-alpha-20/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-21 h1:ig8p6cGK8p34YL1itx+8dXwcEpahcjRQMttOgaJH6Nw= +github.com/erigontech/torrent v1.54.2-alpha-21/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From b0481b6ab62bf697e5b90ff7a98b6558e0cafe99 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sat, 29 Jun 2024 22:25:29 +0100 Subject: [PATCH 27/38] put printer --- erigon-lib/downloader/mdbx_piece_completion.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 2aef159ba3f..1e29b541d92 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -19,6 +19,7 @@ package downloader import ( "context" "encoding/binary" + "fmt" "sync" "github.com/RoaringBitmap/roaring" @@ -178,6 +179,7 @@ func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushe if setters.GetCardinality() > 0 { setters.Iterate(func(piece uint32) bool { + fmt.Println("PUT", infoHash, piece) // TODO deal with error (? don't remove from bitset ?) _ = putCompletion(tx, infoHash, piece, true) return true From 5890ab2de411d53ba3924ab3e174d7237a0951b0 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 14:27:30 +0100 Subject: [PATCH 28/38] flush printer --- erigon-lib/downloader/mdbx_piece_completion.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 1e29b541d92..76bc777872d 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -173,6 +173,11 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm } func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { + flushed.Iterate(func(piece uint32) bool { + fmt.Println("FLUSH", infoHash, piece) + return true + }) + if completed, ok := m.completed[infoHash]; ok { setters := flushed.Clone() setters.And(completed) From 9f26fb3a8308330a111197f42d986d8853206f53 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 16:09:43 +0100 Subject: [PATCH 29/38] remove flushed printer --- erigon-lib/downloader/mdbx_piece_completion.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 76bc777872d..1e29b541d92 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -173,11 +173,6 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm } func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { - flushed.Iterate(func(piece uint32) bool { - fmt.Println("FLUSH", infoHash, piece) - return true - }) - if completed, ok := m.completed[infoHash]; ok { setters := flushed.Clone() setters.And(completed) From 614ee7c500e3581517f30427cfec75cefe6fdc74 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 17:52:00 +0100 Subject: [PATCH 30/38] flush printer --- erigon-lib/downloader/mdbx_piece_completion.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 1e29b541d92..f08e3831079 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -173,6 +173,11 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm } func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { + flushed.Iterate(func(piece uint32) bool { + fmt.Println("FLSH", infoHash, piece) + return true + }) + if completed, ok := m.completed[infoHash]; ok { setters := flushed.Clone() setters.And(completed) From 8ca9ce193e403beef42bdc9677d4c3cde6347ccd Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 18:33:56 +0100 Subject: [PATCH 31/38] added flush set --- erigon-lib/downloader/mdbx_piece_completion.go | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index f08e3831079..469c37606c6 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -40,6 +40,7 @@ type mdbxPieceCompletion struct { db *mdbx.MdbxKV mu sync.RWMutex completed map[infohash.T]*roaring.Bitmap + flushed map[infohash.T]*roaring.Bitmap logger log.Logger } @@ -118,7 +119,9 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { completed.Add(uint32(pk.Index)) - return nil + if flushed, ok := m.completed[pk.InfoHash]; !ok || !flushed.Contains(uint32(pk.Index)) { + return nil + } } tx, err = m.db.BeginRw(context.Background()) @@ -138,6 +141,7 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { } func putCompletion(tx kv.RwTx, infoHash infohash.T, index uint32, c bool) error { + fmt.Println("PUT", infoHash, index) var key [infohash.Size + 4]byte copy(key[:], infoHash[:]) binary.BigEndian.PutUint32(key[infohash.Size:], index) @@ -184,7 +188,6 @@ func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushe if setters.GetCardinality() > 0 { setters.Iterate(func(piece uint32) bool { - fmt.Println("PUT", infoHash, piece) // TODO deal with error (? don't remove from bitset ?) _ = putCompletion(tx, infoHash, piece, true) return true @@ -197,6 +200,15 @@ func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushe delete(m.completed, infoHash) } } + + allFlushed, ok := m.flushed[infoHash] + + if !ok { + allFlushed = &roaring.Bitmap{} + m.completed[infoHash] = allFlushed + } + + allFlushed.Or(flushed) } func (m *mdbxPieceCompletion) Close() error { From fa30ca398f7676de6c2624bce7162587b8a6efa6 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 18:40:32 +0100 Subject: [PATCH 32/38] added flush set --- erigon-lib/downloader/mdbx_piece_completion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 469c37606c6..ad33dcb3481 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -119,7 +119,7 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { completed.Add(uint32(pk.Index)) - if flushed, ok := m.completed[pk.InfoHash]; !ok || !flushed.Contains(uint32(pk.Index)) { + if flushed, ok := m.flushed[pk.InfoHash]; !ok || !flushed.Contains(uint32(pk.Index)) { return nil } } From ac1e2466ff0f1c20c3ed37aaa9a3d43d19f4e993 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 20:59:40 +0100 Subject: [PATCH 33/38] added flush set --- erigon-lib/downloader/mdbx_piece_completion.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index ad33dcb3481..9d6c3b8a31a 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -205,7 +205,7 @@ func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushe if !ok { allFlushed = &roaring.Bitmap{} - m.completed[infoHash] = allFlushed + m.flushed[infoHash] = allFlushed } allFlushed.Or(flushed) From dae06421b8faaf7c05b6cdd310290c6a8ae28c4b Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 21:36:03 +0100 Subject: [PATCH 34/38] add map --- erigon-lib/downloader/mdbx_piece_completion.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 9d6c3b8a31a..2623f7af0ab 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -50,7 +50,8 @@ func NewMdbxPieceCompletion(db kv.RwDB, logger log.Logger) (ret storage.PieceCom ret = &mdbxPieceCompletion{ db: db.(*mdbx.MdbxKV), logger: logger, - completed: map[infohash.T]*roaring.Bitmap{}} + completed: map[infohash.T]*roaring.Bitmap{}, + flushed: map[infohash.T]*roaring.Bitmap{}} return } From b708cfa57c4c35d612393b735436607e5d532136 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Sun, 30 Jun 2024 22:06:46 +0100 Subject: [PATCH 35/38] remove printers --- erigon-lib/downloader/mdbx_piece_completion.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/erigon-lib/downloader/mdbx_piece_completion.go b/erigon-lib/downloader/mdbx_piece_completion.go index 2623f7af0ab..849cdcea20a 100644 --- a/erigon-lib/downloader/mdbx_piece_completion.go +++ b/erigon-lib/downloader/mdbx_piece_completion.go @@ -19,7 +19,6 @@ package downloader import ( "context" "encoding/binary" - "fmt" "sync" "github.com/RoaringBitmap/roaring" @@ -142,7 +141,6 @@ func (m *mdbxPieceCompletion) Set(pk metainfo.PieceKey, b bool) error { } func putCompletion(tx kv.RwTx, infoHash infohash.T, index uint32, c bool) error { - fmt.Println("PUT", infoHash, index) var key [infohash.Size + 4]byte copy(key[:], infoHash[:]) binary.BigEndian.PutUint32(key[infohash.Size:], index) @@ -178,11 +176,6 @@ func (m *mdbxPieceCompletion) Flushed(infoHash infohash.T, flushed *roaring.Bitm } func (m *mdbxPieceCompletion) putFlushed(tx kv.RwTx, infoHash infohash.T, flushed *roaring.Bitmap) { - flushed.Iterate(func(piece uint32) bool { - fmt.Println("FLSH", infoHash, piece) - return true - }) - if completed, ok := m.completed[infoHash]; ok { setters := flushed.Clone() setters.And(completed) From b3675f42a67b257ba9371b1799875f9adedc6602 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 1 Jul 2024 13:06:27 +0100 Subject: [PATCH 36/38] updated erigontech/torrent to v1.54.2-alpha-22 --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index 81ec8c0a97a..d52788b47f0 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -152,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-21 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-22 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 745edb93e75..381be821dff 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-21 h1:ig8p6cGK8p34YL1itx+8dXwcEpahcjRQMttOgaJH6Nw= -github.com/erigontech/torrent v1.54.2-alpha-21/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-22 h1:9DAQLG6gcn13FGKhXZRs2xk3CIkJjXyZf/IqloIfOjw= +github.com/erigontech/torrent v1.54.2-alpha-22/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index 1bb3512c1ad..d5a6f817ddb 100644 --- a/go.mod +++ b/go.mod @@ -296,6 +296,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-21 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-22 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 7b689994391..1133219dc07 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-21 h1:ig8p6cGK8p34YL1itx+8dXwcEpahcjRQMttOgaJH6Nw= -github.com/erigontech/torrent v1.54.2-alpha-21/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-22 h1:9DAQLG6gcn13FGKhXZRs2xk3CIkJjXyZf/IqloIfOjw= +github.com/erigontech/torrent v1.54.2-alpha-22/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From 66b2282bca10b6730bb9ebfe6fa4bad06b6bec43 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 1 Jul 2024 19:58:16 +0100 Subject: [PATCH 37/38] updated erigontech/torrent to v1.54.2-alpha-23 --- erigon-lib/go.mod | 2 +- erigon-lib/go.sum | 4 ++-- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index d52788b47f0..a7e1fa80399 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -152,7 +152,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-22 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-23 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 381be821dff..2d7dac44d0b 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-22 h1:9DAQLG6gcn13FGKhXZRs2xk3CIkJjXyZf/IqloIfOjw= -github.com/erigontech/torrent v1.54.2-alpha-22/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-23 h1:imXee2iTTM8nRTaYyMFDdPvsLPvPJvfWWV+AC7E8yFY= +github.com/erigontech/torrent v1.54.2-alpha-23/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= diff --git a/go.mod b/go.mod index d5a6f817ddb..0714f534bca 100644 --- a/go.mod +++ b/go.mod @@ -296,6 +296,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-22 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-23 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 1133219dc07..7a2200ecc4c 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-22 h1:9DAQLG6gcn13FGKhXZRs2xk3CIkJjXyZf/IqloIfOjw= -github.com/erigontech/torrent v1.54.2-alpha-22/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-23 h1:imXee2iTTM8nRTaYyMFDdPvsLPvPJvfWWV+AC7E8yFY= +github.com/erigontech/torrent v1.54.2-alpha-23/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= From fe0fd6da4f7872ed914a992eaf04d6a53b74b706 Mon Sep 17 00:00:00 2001 From: Mark Holt Date: Mon, 1 Jul 2024 20:36:59 +0100 Subject: [PATCH 38/38] updated erigontech/torrent to v1.54.2-alpha-24 --- erigon-lib/go.mod | 4 +--- erigon-lib/go.sum | 8 ++------ go.mod | 4 +--- go.sum | 8 ++------ 4 files changed, 6 insertions(+), 18 deletions(-) diff --git a/erigon-lib/go.mod b/erigon-lib/go.mod index a7e1fa80399..ddbba5b00ac 100644 --- a/erigon-lib/go.mod +++ b/erigon-lib/go.mod @@ -54,9 +54,7 @@ require ( github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/ianlancetaylor/cgosymbolizer v0.0.0-20240503222823-736c933a666d // indirect github.com/opencontainers/runtime-spec v1.2.0 // indirect - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect github.com/pion/udp v0.1.4 // indirect - github.com/sasha-s/go-deadlock v0.3.1 // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240515191416-fc5f0ca64291 // indirect @@ -152,7 +150,7 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-23 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-24 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 github.com/tidwall/btree => github.com/AskAlexSharov/btree v1.6.2 ) diff --git a/erigon-lib/go.sum b/erigon-lib/go.sum index 2d7dac44d0b..f9f6fbc4abc 100644 --- a/erigon-lib/go.sum +++ b/erigon-lib/go.sum @@ -146,8 +146,8 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBgrQLo= github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= -github.com/erigontech/torrent v1.54.2-alpha-23 h1:imXee2iTTM8nRTaYyMFDdPvsLPvPJvfWWV+AC7E8yFY= -github.com/erigontech/torrent v1.54.2-alpha-23/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-24 h1:LNfGFxRQHSND/r/rQ2ibFU2ghPGLJq41ZQdF5uRY8NY= +github.com/erigontech/torrent v1.54.2-alpha-24/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= @@ -326,8 +326,6 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= @@ -420,8 +418,6 @@ github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= -github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= github.com/shirou/gopsutil/v3 v3.24.4 h1:dEHgzZXt4LMNm+oYELpzl9YCqV65Yr/6SfrvgRBtXeU= github.com/shirou/gopsutil/v3 v3.24.4/go.mod h1:lTd2mdiOspcqLgAnr9/nGi71NkeMpWKdmhuxm9GusH8= diff --git a/go.mod b/go.mod index 0714f534bca..8d532ebcfc6 100644 --- a/go.mod +++ b/go.mod @@ -115,8 +115,6 @@ require ( github.com/go-ole/go-ole v1.2.6 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-retryablehttp v0.7.6 // indirect - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect - github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/tklauser/go-sysconf v0.3.14 // indirect github.com/tklauser/numcpus v0.8.0 // indirect github.com/yusufpapurcu/wmi v1.2.4 // indirect @@ -296,6 +294,6 @@ require ( ) replace ( - github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-23 + github.com/anacrolix/torrent => github.com/erigontech/torrent v1.54.2-alpha-24 github.com/holiman/bloomfilter/v2 => github.com/AskAlexSharov/bloomfilter/v2 v2.0.8 ) diff --git a/go.sum b/go.sum index 7a2200ecc4c..31a935303f4 100644 --- a/go.sum +++ b/go.sum @@ -277,8 +277,8 @@ github.com/erigontech/mdbx-go v0.38.4 h1:S9T7mTe9KPcFe4dOoOtVdI6gPzht9y7wMnYfUBg github.com/erigontech/mdbx-go v0.38.4/go.mod h1:IcOLQDPw3VM/asP6T5JVPPN4FHHgJtY16XfYjzWKVNI= github.com/erigontech/silkworm-go v0.18.0 h1:j56p61xZHBFhZGH1OixlGU8KcfjHzcw9pjAfjmVsOZA= github.com/erigontech/silkworm-go v0.18.0/go.mod h1:O50ux0apICEVEGyRWiE488K8qz8lc3PA/SXbQQAc8SU= -github.com/erigontech/torrent v1.54.2-alpha-23 h1:imXee2iTTM8nRTaYyMFDdPvsLPvPJvfWWV+AC7E8yFY= -github.com/erigontech/torrent v1.54.2-alpha-23/go.mod h1:PYTzOlX3lVcdIOITLnSdGvqARQDa8WQ2XPofZzeaDqo= +github.com/erigontech/torrent v1.54.2-alpha-24 h1:LNfGFxRQHSND/r/rQ2ibFU2ghPGLJq41ZQdF5uRY8NY= +github.com/erigontech/torrent v1.54.2-alpha-24/go.mod h1:QtK2WLdEz1Iy1Dh/325UltdHU0nA1xujh2rN6aov6y0= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/fjl/gencodec v0.0.0-20220412091415-8bb9e558978c h1:CndMRAH4JIwxbW8KYq6Q+cGWcGHz0FjGR3QqcInWcW0= @@ -680,8 +680,6 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.6 h1:1IxKJntfSlYkpUj8LlYRSWpYiTTC02nUrOE8T3DqGeg= @@ -805,8 +803,6 @@ github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= -github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= -github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=