diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index ced95a5dc9..d4c229eb36 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -3,7 +3,7 @@ name: Go on: push: pull_request: - branches: [ master ] + branches: [ main ] env: CGO_CFLAGS_ALLOW: "-O -D__BLST_PORTABLE__" diff --git a/README.md b/README.md index 185b702ecb..1b9be011eb 100644 --- a/README.md +++ b/README.md @@ -35,26 +35,69 @@ Configure geth for your network, it will become the block builder. Builder-related options: ``` $ geth --help -BUILDER API OPTIONS: + + BUILDER + --builder (default: false) Enable the builder + --builder.beacon_endpoint value (default: "http://127.0.0.1:5052") + Beacon endpoint to connect to for beacon chain data [$BUILDER_BEACON_ENDPOINT] + --builder.bellatrix_fork_version value (default: "0x02000000") + Bellatrix fork version. [$BUILDER_BELLATRIX_FORK_VERSION] + --builder.dry-run (default: false) + Builder only validates blocks without submission to the relay + --builder.genesis_fork_version value (default: "0x00000000") + Gensis fork version. [$BUILDER_GENESIS_FORK_VERSION] + --builder.genesis_validators_root value (default: "0x0000000000000000000000000000000000000000000000000000000000000000") + Genesis validators root of the network. [$BUILDER_GENESIS_VALIDATORS_ROOT] + --builder.listen_addr value (default: ":28545") Listening address for builder endpoint [$BUILDER_LISTEN_ADDR] + --builder.local_relay (default: false) + Enable the local relay + --builder.no_bundle_fetcher (default: false) + Disable the bundle fetcher + --builder.relay_secret_key value (default: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11") + Builder local relay API key used for signing headers [$BUILDER_RELAY_SECRET_KEY] + --builder.remote_relay_endpoint value + Relay endpoint to connect to for validator registration data, if not provided + will expose validator registration locally [$BUILDER_REMOTE_RELAY_ENDPOINT] + + --builder.secondary_remote_relay_endpoints value + Comma separated relay endpoints to connect to for validator registration data + missing from the primary remote relay, and to push blocks for registrations + missing from or matching the primary [$BUILDER_SECONDARY_REMOTE_RELAY_ENDPOINTS] + --builder.secret_key value (default: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11") + Builder key used for signing blocks [$BUILDER_SECRET_KEY] + + --builder.validation_blacklist value (default: "ofac_blacklist.json") + Path to file containing blacklisted addresses, json-encoded list of strings. + Default assumes CWD is repo's root + --builder.validator_checks (default: false) - --builder.validation_blacklist value + Enable the validator checks + + MINER + --miner.algotype value (default: "mev-geth") - --miner.blocklist value - --miner.extradata value + Block building algorithm to use [=mev-geth] (mev-geth, greedy) + + --miner.blocklist value + flashbots - Path to JSON file with list of blocked addresses. Miner will ignore + txs that touch mentioned addresses. + + --miner.extradata value + Block extra data set by the miner (default = client version) ``` Environment variables: diff --git a/builder/builder.go b/builder/builder.go index 7dde6fe325..0c86eb7846 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -37,7 +37,7 @@ type IBeaconClient interface { } type IRelay interface { - SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest) error + SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest, vd ValidatorData) error GetValidatorForSlot(nextSlot uint64) (ValidatorData, error) } @@ -97,7 +97,7 @@ func (b *Builder) Stop() error { return nil } -func (b *Builder) onSealedBlock(block *types.Block, ordersClosedAt time.Time, sealedAt time.Time, commitedBundles []types.SimulatedBundle, allBundles []types.SimulatedBundle, proposerPubkey boostTypes.PublicKey, proposerFeeRecipient boostTypes.Address, proposerRegisteredGasLimit uint64, attrs *BuilderPayloadAttributes) error { +func (b *Builder) onSealedBlock(block *types.Block, ordersClosedAt time.Time, sealedAt time.Time, commitedBundles []types.SimulatedBundle, allBundles []types.SimulatedBundle, proposerPubkey boostTypes.PublicKey, vd ValidatorData, attrs *BuilderPayloadAttributes) error { executableData := beacon.BlockToExecutableData(block) payload, err := executableDataToExecutionPayload(executableData) if err != nil { @@ -118,7 +118,7 @@ func (b *Builder) onSealedBlock(block *types.Block, ordersClosedAt time.Time, se BlockHash: payload.BlockHash, BuilderPubkey: b.builderPublicKey, ProposerPubkey: proposerPubkey, - ProposerFeeRecipient: proposerFeeRecipient, + ProposerFeeRecipient: vd.FeeRecipient, GasLimit: executableData.GasLimit, GasUsed: executableData.GasUsed, Value: *value, @@ -137,13 +137,13 @@ func (b *Builder) onSealedBlock(block *types.Block, ordersClosedAt time.Time, se } if b.dryRun { - err = b.validator.ValidateBuilderSubmissionV1(&blockvalidation.BuilderBlockValidationRequest{blockSubmitReq, proposerRegisteredGasLimit}) + err = b.validator.ValidateBuilderSubmissionV1(&blockvalidation.BuilderBlockValidationRequest{blockSubmitReq, vd.GasLimit}) if err != nil { log.Error("could not validate block", "err", err) } } else { go b.ds.ConsumeBuiltBlock(block, ordersClosedAt, sealedAt, commitedBundles, allBundles, &blockBidMsg) - err = b.relay.SubmitBlock(&blockSubmitReq) + err = b.relay.SubmitBlock(&blockSubmitReq, vd) if err != nil { log.Error("could not submit block", "err", err, "#commitedBundles", len(commitedBundles)) return err @@ -208,7 +208,7 @@ func (b *Builder) OnPayloadAttribute(attrs *BuilderPayloadAttributes) error { } b.slotAttrs = append(b.slotAttrs, *attrs) - go b.runBuildingJob(b.slotCtx, proposerPubkey, vd.FeeRecipient, vd.GasLimit, attrs) + go b.runBuildingJob(b.slotCtx, proposerPubkey, vd, attrs) return nil } @@ -220,7 +220,7 @@ type blockQueueEntry struct { allBundles []types.SimulatedBundle } -func (b *Builder) runBuildingJob(slotCtx context.Context, proposerPubkey boostTypes.PublicKey, feeRecipient boostTypes.Address, proposerRegisteredGasLimit uint64, attrs *BuilderPayloadAttributes) { +func (b *Builder) runBuildingJob(slotCtx context.Context, proposerPubkey boostTypes.PublicKey, vd ValidatorData, attrs *BuilderPayloadAttributes) { ctx, cancel := context.WithTimeout(slotCtx, 12*time.Second) defer cancel() @@ -245,7 +245,7 @@ func (b *Builder) runBuildingJob(slotCtx context.Context, proposerPubkey boostTy submitBestBlock := func() { queueMu.Lock() if queueLastSubmittedProfit.Cmp(queueBestProfit) < 0 { - err := b.onSealedBlock(queueBestEntry.block, queueBestEntry.ordersCloseTime, queueBestEntry.sealedAt, queueBestEntry.commitedBundles, queueBestEntry.allBundles, proposerPubkey, feeRecipient, proposerRegisteredGasLimit, attrs) + err := b.onSealedBlock(queueBestEntry.block, queueBestEntry.ordersCloseTime, queueBestEntry.sealedAt, queueBestEntry.commitedBundles, queueBestEntry.allBundles, proposerPubkey, vd, attrs) if err != nil { log.Error("could not run sealed block hook", "err", err) diff --git a/builder/builder_test.go b/builder/builder_test.go index 09d4c5b728..2069252b5a 100644 --- a/builder/builder_test.go +++ b/builder/builder_test.go @@ -30,7 +30,7 @@ func TestOnPayloadAttributes(t *testing.T) { feeRecipient, _ := boostTypes.HexToAddress("0xabcf8e0d4e9587369b2301d0790347320302cc00") testRelay := testRelay{ - validator: ValidatorData{ + gvsVd: ValidatorData{ Pubkey: PubkeyHex(testBeacon.validator.Pk.String()), FeeRecipient: feeRecipient, GasLimit: 10, diff --git a/builder/config.go b/builder/config.go index facde13f67..e22d0c3817 100644 --- a/builder/config.go +++ b/builder/config.go @@ -1,36 +1,38 @@ package builder type Config struct { - Enabled bool `toml:",omitempty"` - EnableValidatorChecks bool `toml:",omitempty"` - EnableLocalRelay bool `toml:",omitempty"` - DisableBundleFetcher bool `toml:",omitempty"` - DryRun bool `toml:",omitempty"` - BuilderSecretKey string `toml:",omitempty"` - RelaySecretKey string `toml:",omitempty"` - ListenAddr string `toml:",omitempty"` - GenesisForkVersion string `toml:",omitempty"` - BellatrixForkVersion string `toml:",omitempty"` - GenesisValidatorsRoot string `toml:",omitempty"` - BeaconEndpoint string `toml:",omitempty"` - RemoteRelayEndpoint string `toml:",omitempty"` - ValidationBlocklist string `toml:",omitempty"` + Enabled bool `toml:",omitempty"` + EnableValidatorChecks bool `toml:",omitempty"` + EnableLocalRelay bool `toml:",omitempty"` + DisableBundleFetcher bool `toml:",omitempty"` + DryRun bool `toml:",omitempty"` + BuilderSecretKey string `toml:",omitempty"` + RelaySecretKey string `toml:",omitempty"` + ListenAddr string `toml:",omitempty"` + GenesisForkVersion string `toml:",omitempty"` + BellatrixForkVersion string `toml:",omitempty"` + GenesisValidatorsRoot string `toml:",omitempty"` + BeaconEndpoint string `toml:",omitempty"` + RemoteRelayEndpoint string `toml:",omitempty"` + SecondaryRemoteRelayEndpoints []string `toml:",omitempty"` + ValidationBlocklist string `toml:",omitempty"` } // DefaultConfig is the default config for the builder. var DefaultConfig = Config{ - Enabled: false, - EnableValidatorChecks: false, - EnableLocalRelay: false, - DisableBundleFetcher: false, - DryRun: false, - BuilderSecretKey: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", - RelaySecretKey: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", - ListenAddr: ":28545", - GenesisForkVersion: "0x00000000", - BellatrixForkVersion: "0x02000000", - GenesisValidatorsRoot: "0x0000000000000000000000000000000000000000000000000000000000000000", - BeaconEndpoint: "http://127.0.0.1:5052", - RemoteRelayEndpoint: "", - ValidationBlocklist: "", + Enabled: false, + EnableValidatorChecks: false, + EnableLocalRelay: false, + DisableBundleFetcher: false, + DryRun: false, + BuilderSecretKey: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + RelaySecretKey: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + ListenAddr: ":28545", + GenesisForkVersion: "0x00000000", + BellatrixForkVersion: "0x02000000", + GenesisValidatorsRoot: "0x0000000000000000000000000000000000000000000000000000000000000000", + BeaconEndpoint: "http://127.0.0.1:5052", + RemoteRelayEndpoint: "", + SecondaryRemoteRelayEndpoints: nil, + ValidationBlocklist: "", } diff --git a/builder/local_relay.go b/builder/local_relay.go index 68ec8f91f1..423c4ebe91 100644 --- a/builder/local_relay.go +++ b/builder/local_relay.go @@ -79,7 +79,12 @@ func NewLocalRelay(sk *bls.SecretKey, beaconClient IBeaconClient, builderSigning } } -func (r *LocalRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest) error { +func (r *LocalRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest, _ ValidatorData) error { + log.Info("submitting block to local relay", "block", msg.ExecutionPayload.BlockHash.String()) + return r.submitBlock(msg) +} + +func (r *LocalRelay) submitBlock(msg *boostTypes.BuilderSubmitBlockRequest) error { payloadHeader, err := boostTypes.PayloadToPayloadHeader(msg.ExecutionPayload) if err != nil { log.Error("could not convert payload to header", "err", err) diff --git a/builder/relay.go b/builder/relay.go index f5414ae280..5144a12d4e 100644 --- a/builder/relay.go +++ b/builder/relay.go @@ -15,20 +15,7 @@ import ( "github.com/flashbots/mev-boost/server" ) -type testRelay struct { - validator ValidatorData - requestedSlot uint64 - submittedMsg *boostTypes.BuilderSubmitBlockRequest -} - -func (r *testRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest) error { - r.submittedMsg = msg - return nil -} -func (r *testRelay) GetValidatorForSlot(nextSlot uint64) (ValidatorData, error) { - r.requestedSlot = nextSlot - return r.validator, nil -} +var ErrValidatorNotFound = errors.New("validator not found") type RemoteRelay struct { endpoint string @@ -135,20 +122,21 @@ func (r *RemoteRelay) GetValidatorForSlot(nextSlot uint64) (ValidatorData, error return vd, nil } - return ValidatorData{}, errors.New("validator not found") + return ValidatorData{}, ErrValidatorNotFound } -func (r *RemoteRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest) error { +func (r *RemoteRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest, _ ValidatorData) error { + log.Info("submitting block to remote relay", "endpoint", r.endpoint) code, err := server.SendHTTPRequest(context.TODO(), *http.DefaultClient, http.MethodPost, r.endpoint+"/relay/v1/builder/blocks", msg, nil) if err != nil { - return err + return fmt.Errorf("error sending http request to relay %s. err: %w", r.endpoint, err) } if code > 299 { - return fmt.Errorf("non-ok response code %d from relay ", code) + return fmt.Errorf("non-ok response code %d from relay %s", code, r.endpoint) } if r.localRelay != nil { - r.localRelay.SubmitBlock(msg) + r.localRelay.submitBlock(msg) } return nil diff --git a/builder/relay_aggregator.go b/builder/relay_aggregator.go new file mode 100644 index 0000000000..7071791f09 --- /dev/null +++ b/builder/relay_aggregator.go @@ -0,0 +1,124 @@ +package builder + +import ( + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/log" + boostTypes "github.com/flashbots/go-boost-utils/types" +) + +type RemoteRelayAggregator struct { + relays []IRelay // in order of precedence, primary first + + registrationsCacheLock sync.RWMutex + registrationsCacheSlot uint64 + registrationsCache map[ValidatorData][]IRelay +} + +func NewRemoteRelayAggregator(primary IRelay, secondary []IRelay) *RemoteRelayAggregator { + relays := []IRelay{primary} + return &RemoteRelayAggregator{ + relays: append(relays, secondary...), + } +} + +func (r *RemoteRelayAggregator) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest, registration ValidatorData) error { + r.registrationsCacheLock.RLock() + defer r.registrationsCacheLock.RUnlock() + + relays, found := r.registrationsCache[registration] + if !found { + return fmt.Errorf("no relays for registration %s", registration.Pubkey) + } + for _, relay := range relays { + go func(relay IRelay) { + err := relay.SubmitBlock(msg, registration) + if err != nil { + log.Error("could not submit block", "err", err) + } + }(relay) + } + + return nil +} + +type RelayValidatorRegistration struct { + vd ValidatorData + relayI int // index into relays array to preserve relative order +} + +func (r *RemoteRelayAggregator) GetValidatorForSlot(nextSlot uint64) (ValidatorData, error) { + registrationsCh := make(chan *RelayValidatorRegistration, len(r.relays)) + for i, relay := range r.relays { + go func(relay IRelay, relayI int) { + vd, err := relay.GetValidatorForSlot(nextSlot) + if err == nil { + registrationsCh <- &RelayValidatorRegistration{vd: vd, relayI: relayI} + } else if errors.Is(err, ErrValidatorNotFound) { + registrationsCh <- nil + } else { + log.Error("could not get validator registration", "err", err) + registrationsCh <- nil + } + }(relay, i) + } + + topRegistrationCh := make(chan ValidatorData, 1) + go r.updateRelayRegistrations(nextSlot, registrationsCh, topRegistrationCh) + + if vd, ok := <-topRegistrationCh; ok { + return vd, nil + } + return ValidatorData{}, ErrValidatorNotFound +} + +func (r *RemoteRelayAggregator) updateRelayRegistrations(nextSlot uint64, registrationsCh chan *RelayValidatorRegistration, topRegistrationCh chan ValidatorData) { + defer close(topRegistrationCh) + + r.registrationsCacheLock.Lock() + defer r.registrationsCacheLock.Unlock() + + if nextSlot < r.registrationsCacheSlot { + // slot is outdated + return + } + + registrations := make([]*RelayValidatorRegistration, 0, len(r.relays)) + bestRelayIndex := len(r.relays) // relay index of the topmost relay that gave us the registration + bestRelayRegistrationIndex := -1 // index into the registrations for the registration returned by topmost relay + for i := 0; i < len(r.relays); i++ { + relayRegistration := <-registrationsCh + if relayRegistration != nil { + registrations = append(registrations, relayRegistration) + // happy path for primary + if relayRegistration.relayI == 0 { + topRegistrationCh <- relayRegistration.vd + } + if relayRegistration.relayI < bestRelayIndex { + bestRelayIndex = relayRegistration.relayI + bestRelayRegistrationIndex = len(registrations) - 1 + } + } + } + + if len(registrations) == 0 { + return + } + + if bestRelayIndex != 0 { + // if bestRelayIndex == 0 it was already sent + topRegistrationCh <- registrations[bestRelayRegistrationIndex].vd + } + + if nextSlot > r.registrationsCacheSlot { + // clear the cache + r.registrationsCache = make(map[ValidatorData][]IRelay) + r.registrationsCacheSlot = nextSlot + } + + for _, relayRegistration := range registrations { + r.registrationsCache[relayRegistration.vd] = append(r.registrationsCache[relayRegistration.vd], r.relays[relayRegistration.relayI]) + } +} diff --git a/builder/relay_aggregator_test.go b/builder/relay_aggregator_test.go new file mode 100644 index 0000000000..5465a6d00b --- /dev/null +++ b/builder/relay_aggregator_test.go @@ -0,0 +1,214 @@ +package builder + +import ( + "errors" + "testing" + "time" + + boostTypes "github.com/flashbots/go-boost-utils/types" + "github.com/stretchr/testify/require" +) + +/* + validator ValidatorData + requestedSlot uint64 + submittedMsg *boostTypes.BuilderSubmitBlockRequest +*/ + +type testRelay struct { + sbError error + gvsVd ValidatorData + gvsErr error + + requestedSlot uint64 + submittedMsg *boostTypes.BuilderSubmitBlockRequest + submittedMsgCh chan *boostTypes.BuilderSubmitBlockRequest + submittedRegistration ValidatorData +} + +type testRelayAggBackend struct { + relays []*testRelay + ragg *RemoteRelayAggregator +} + +func newTestRelayAggBackend(numRelay int) *testRelayAggBackend { + testRelays := make([]*testRelay, numRelay) + secondaryRelays := make([]IRelay, numRelay-1) + for i := 0; i < numRelay; i++ { + testRelays[i] = &testRelay{} + if i > 0 { + secondaryRelays[i-1] = testRelays[i] + } + } + ragg := NewRemoteRelayAggregator(testRelays[0], secondaryRelays) + return &testRelayAggBackend{testRelays, ragg} +} + +func (r *testRelay) SubmitBlock(msg *boostTypes.BuilderSubmitBlockRequest, registration ValidatorData) error { + if r.submittedMsgCh != nil { + select { + case r.submittedMsgCh <- msg: + default: + } + } + r.submittedMsg = msg + return r.sbError +} +func (r *testRelay) GetValidatorForSlot(nextSlot uint64) (ValidatorData, error) { + r.requestedSlot = nextSlot + return r.gvsVd, r.gvsErr +} + +func TestRemoteRelayAggregator(t *testing.T) { + t.Run("should return error if no relays return validator data", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + // make all error out + for _, r := range backend.relays { + r.gvsErr = errors.New("error!") + } + + // Check getting validator slot - should error out if no relays return + _, err := backend.ragg.GetValidatorForSlot(10) + require.Error(t, ErrValidatorNotFound, err) + }) + + t.Run("should return validator if one relay returns validator data", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + + // If primary returns should not error out + backend.relays[1].gvsErr = errors.New("error!") + backend.relays[2].gvsErr = errors.New("error!") + _, err := backend.ragg.GetValidatorForSlot(10) + require.NoError(t, err) + + // If any returns should not error out + backend.relays[0].gvsErr = errors.New("error!") + backend.relays[2].gvsErr = nil + _, err = backend.ragg.GetValidatorForSlot(10) + require.NoError(t, err) + }) + + t.Run("should return the more important (lower index) relay validator data", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + + // Should return the more important relay if primary fails + backend.relays[0].gvsErr = errors.New("error!") + backend.relays[1].gvsVd.GasLimit = 20 + backend.relays[2].gvsVd.GasLimit = 30 + vd, err := backend.ragg.GetValidatorForSlot(10) + require.NoError(t, err) + require.Equal(t, uint64(20), vd.GasLimit) + + // Should return the primary if it returns + backend.relays[0].gvsErr = nil + backend.relays[0].gvsVd.GasLimit = 10 + vd, err = backend.ragg.GetValidatorForSlot(11) + require.NoError(t, err) + require.Equal(t, uint64(10), vd.GasLimit) + }) + + t.Run("should error submitting to unseen validator data", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + + backend.relays[0].gvsVd.GasLimit = 10 + + vd, err := backend.ragg.GetValidatorForSlot(10) + require.NoError(t, err) + require.Equal(t, uint64(10), vd.GasLimit) + + // let the validator registrations finish + // TODO: notify from the test relays + time.Sleep(10 * time.Millisecond) + + // if submitting for unseen VD should error out + msg := &boostTypes.BuilderSubmitBlockRequest{} + err = backend.ragg.SubmitBlock(msg, ValidatorData{GasLimit: 40}) + require.Error(t, err) + }) + + t.Run("should submit to relay with matching validator data", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + + backend.relays[0].gvsVd.GasLimit = 10 + backend.relays[1].gvsVd.GasLimit = 20 + backend.relays[2].gvsVd.GasLimit = 30 + + vd, err := backend.ragg.GetValidatorForSlot(11) + require.NoError(t, err) + require.Equal(t, uint64(10), vd.GasLimit) + + // let the validator registrations finish + // TODO: notify from the test relays + time.Sleep(10 * time.Millisecond) + + // if submitting for unseen VD should error out + msg := &boostTypes.BuilderSubmitBlockRequest{} + err = backend.ragg.SubmitBlock(msg, ValidatorData{GasLimit: 40}) + require.Error(t, err) + + // should submit to the single pirmary if its the only one matching + backend.relays[0].submittedMsgCh = make(chan *boostTypes.BuilderSubmitBlockRequest, 1) + err = backend.ragg.SubmitBlock(msg, ValidatorData{GasLimit: 10}) + require.NoError(t, err) + select { + case rsMsg := <-backend.relays[0].submittedMsgCh: + require.Equal(t, msg, rsMsg) + case <-time.After(time.Second): + t.Fail() + } + + // no other relay should have been asked + require.Nil(t, backend.relays[1].submittedMsg) + require.Nil(t, backend.relays[2].submittedMsg) + }) + + t.Run("should submit to relays with matching validator data and drop registrations on next slot", func(t *testing.T) { + backend := newTestRelayAggBackend(3) + + backend.relays[0].gvsVd.GasLimit = 10 + backend.relays[1].gvsVd.GasLimit = 20 + backend.relays[2].gvsVd.GasLimit = 30 + + vd, err := backend.ragg.GetValidatorForSlot(11) + require.NoError(t, err) + require.Equal(t, uint64(10), vd.GasLimit) + + // let the validator registrations finish + time.Sleep(10 * time.Millisecond) + + backend.relays[0].gvsVd.GasLimit = 30 + backend.relays[1].gvsVd.GasLimit = 20 + backend.relays[2].gvsVd.GasLimit = 30 + + // should drop registrations if asked for the next slot + vd, err = backend.ragg.GetValidatorForSlot(12) + require.NoError(t, err) + require.Equal(t, uint64(30), vd.GasLimit) + + time.Sleep(10 * time.Millisecond) + + // should submit to multiple matching relays + backend.relays[0].submittedMsgCh = make(chan *boostTypes.BuilderSubmitBlockRequest, 1) + backend.relays[2].submittedMsgCh = make(chan *boostTypes.BuilderSubmitBlockRequest, 1) + msg := &boostTypes.BuilderSubmitBlockRequest{} + err = backend.ragg.SubmitBlock(msg, ValidatorData{GasLimit: 10}) + require.Error(t, err) + + err = backend.ragg.SubmitBlock(msg, ValidatorData{GasLimit: 30}) + require.NoError(t, err) + + select { + case rsMsg := <-backend.relays[0].submittedMsgCh: + require.Equal(t, msg, rsMsg) + case <-time.After(time.Second): + t.Fail() + } + + select { + case rsMsg := <-backend.relays[2].submittedMsgCh: + require.Equal(t, msg, rsMsg) + case <-time.After(time.Second): + t.Fail() + } + }) +} diff --git a/builder/service.go b/builder/service.go index beeaa092f7..b8a131c918 100644 --- a/builder/service.go +++ b/builder/service.go @@ -160,6 +160,14 @@ func Register(stack *node.Node, backend *eth.Ethereum, cfg *Config) error { return errors.New("neither local nor remote relay specified") } + if len(cfg.SecondaryRemoteRelayEndpoints) > 0 { + secondaryRelays := make([]IRelay, len(cfg.SecondaryRemoteRelayEndpoints)) + for i, endpoint := range cfg.SecondaryRemoteRelayEndpoints { + secondaryRelays[i] = NewRemoteRelay(endpoint, nil) + } + relay = NewRemoteRelayAggregator(relay, secondaryRelays) + } + var validator *blockvalidation.BlockValidationAPI if cfg.DryRun { var accessVerifier *blockvalidation.AccessVerifier diff --git a/cmd/geth/main.go b/cmd/geth/main.go index df1ded7dd9..320340897d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,13 +158,13 @@ var ( utils.GpoIgnoreGasPriceFlag, utils.MinerNotifyFullFlag, utils.IgnoreLegacyReceiptsFlag, - utils.BuilderBlockValidationBlacklistSourceFilePath, configFileFlag, }, utils.NetworkFlags, utils.DatabasePathFlags) builderApiFlags = []cli.Flag{ utils.BuilderEnabled, utils.BuilderEnableValidatorChecks, + utils.BuilderBlockValidationBlacklistSourceFilePath, utils.BuilderEnableLocalRelay, utils.BuilderDisableBundleFetcher, utils.BuilderDryRun, @@ -176,6 +176,7 @@ var ( utils.BuilderGenesisValidatorsRoot, utils.BuilderBeaconEndpoint, utils.BuilderRemoteRelayEndpoint, + utils.BuilderSecondaryRemoteRelayEndpoints, } rpcFlags = []cli.Flag{ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e26761789e..d44e1e3491 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -688,72 +688,98 @@ var ( } // Builder API settings BuilderEnabled = &cli.BoolFlag{ - Name: "builder", - Usage: "Enable the builder", + Name: "builder", + Usage: "Enable the builder", + Category: flags.BuilderCategory, } BuilderEnableValidatorChecks = &cli.BoolFlag{ - Name: "builder.validator_checks", - Usage: "Enable the validator checks", + Name: "builder.validator_checks", + Usage: "Enable the validator checks", + Category: flags.BuilderCategory, + } + BuilderBlockValidationBlacklistSourceFilePath = &cli.StringFlag{ + Name: "builder.validation_blacklist", + Usage: "Path to file containing blacklisted addresses, json-encoded list of strings. Default assumes CWD is repo's root", + Value: "ofac_blacklist.json", + Category: flags.BuilderCategory, } BuilderEnableLocalRelay = &cli.BoolFlag{ - Name: "builder.local_relay", - Usage: "Enable the local relay", + Name: "builder.local_relay", + Usage: "Enable the local relay", + Category: flags.BuilderCategory, } BuilderDisableBundleFetcher = &cli.BoolFlag{ - Name: "builder.no_bundle_fetcher", - Usage: "Disable the bundle fetcher", + Name: "builder.no_bundle_fetcher", + Usage: "Disable the bundle fetcher", + Category: flags.BuilderCategory, } BuilderDryRun = &cli.BoolFlag{ - Name: "builder.dry-run", - Usage: "Builder only validates blocks without submission to the relay", + Name: "builder.dry-run", + Usage: "Builder only validates blocks without submission to the relay", + Category: flags.BuilderCategory, } BuilderSecretKey = &cli.StringFlag{ - Name: "builder.secret_key", - Usage: "Builder key used for signing blocks", - EnvVars: []string{"BUILDER_SECRET_KEY"}, - Value: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + Name: "builder.secret_key", + Usage: "Builder key used for signing blocks", + EnvVars: []string{"BUILDER_SECRET_KEY"}, + Value: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + Category: flags.BuilderCategory, } BuilderRelaySecretKey = &cli.StringFlag{ - Name: "builder.relay_secret_key", - Usage: "Builder local relay API key used for signing headers", - EnvVars: []string{"BUILDER_RELAY_SECRET_KEY"}, - Value: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + Name: "builder.relay_secret_key", + Usage: "Builder local relay API key used for signing headers", + EnvVars: []string{"BUILDER_RELAY_SECRET_KEY"}, + Value: "0x2fc12ae741f29701f8e30f5de6350766c020cb80768a0ff01e6838ffd2431e11", + Category: flags.BuilderCategory, } BuilderListenAddr = &cli.StringFlag{ - Name: "builder.listen_addr", - Usage: "Listening address for builder endpoint", - EnvVars: []string{"BUILDER_LISTEN_ADDR"}, - Value: ":28545", + Name: "builder.listen_addr", + Usage: "Listening address for builder endpoint", + EnvVars: []string{"BUILDER_LISTEN_ADDR"}, + Value: ":28545", + Category: flags.BuilderCategory, } BuilderGenesisForkVersion = &cli.StringFlag{ - Name: "builder.genesis_fork_version", - Usage: "Gensis fork version. For kiln use 0x70000069", - EnvVars: []string{"BUILDER_GENESIS_FORK_VERSION"}, - Value: "0x00000000", + Name: "builder.genesis_fork_version", + Usage: "Gensis fork version.", + EnvVars: []string{"BUILDER_GENESIS_FORK_VERSION"}, + Value: "0x00000000", + Category: flags.BuilderCategory, } BuilderBellatrixForkVersion = &cli.StringFlag{ - Name: "builder.bellatrix_fork_version", - Usage: "Bellatrix fork version. For kiln use 0x70000071", - EnvVars: []string{"BUILDER_BELLATRIX_FORK_VERSION"}, - Value: "0x02000000", + Name: "builder.bellatrix_fork_version", + Usage: "Bellatrix fork version.", + EnvVars: []string{"BUILDER_BELLATRIX_FORK_VERSION"}, + Value: "0x02000000", + Category: flags.BuilderCategory, } BuilderGenesisValidatorsRoot = &cli.StringFlag{ - Name: "builder.genesis_validators_root", - Usage: "Genesis validators root of the network. For kiln use 0x99b09fcd43e5905236c370f184056bec6e6638cfc31a323b304fc4aa789cb4ad", - EnvVars: []string{"BUILDER_GENESIS_VALIDATORS_ROOT"}, - Value: "0x0000000000000000000000000000000000000000000000000000000000000000", + Name: "builder.genesis_validators_root", + Usage: "Genesis validators root of the network.", + EnvVars: []string{"BUILDER_GENESIS_VALIDATORS_ROOT"}, + Value: "0x0000000000000000000000000000000000000000000000000000000000000000", + Category: flags.BuilderCategory, } BuilderBeaconEndpoint = &cli.StringFlag{ - Name: "builder.beacon_endpoint", - Usage: "Beacon endpoint to connect to for beacon chain data", - EnvVars: []string{"BUILDER_BEACON_ENDPOINT"}, - Value: "http://127.0.0.1:5052", + Name: "builder.beacon_endpoint", + Usage: "Beacon endpoint to connect to for beacon chain data", + EnvVars: []string{"BUILDER_BEACON_ENDPOINT"}, + Value: "http://127.0.0.1:5052", + Category: flags.BuilderCategory, } BuilderRemoteRelayEndpoint = &cli.StringFlag{ - Name: "builder.remote_relay_endpoint", - Usage: "Relay endpoint to connect to for validator registration data, if not provided will expose validator registration locally", - EnvVars: []string{"BUILDER_REMOTE_RELAY_ENDPOINT"}, - Value: "", + Name: "builder.remote_relay_endpoint", + Usage: "Relay endpoint to connect to for validator registration data, if not provided will expose validator registration locally", + EnvVars: []string{"BUILDER_REMOTE_RELAY_ENDPOINT"}, + Value: "", + Category: flags.BuilderCategory, + } + BuilderSecondaryRemoteRelayEndpoints = &cli.StringFlag{ + Name: "builder.secondary_remote_relay_endpoints", + Usage: "Comma separated relay endpoints to connect to for validator registration data missing from the primary remote relay, and to push blocks for registrations missing from or matching the primary", + EnvVars: []string{"BUILDER_SECONDARY_REMOTE_RELAY_ENDPOINTS"}, + Value: "", + Category: flags.BuilderCategory, } // RPC settings IPCDisabledFlag = &cli.BoolFlag{ @@ -1071,14 +1097,6 @@ var ( Value: metrics.DefaultConfig.InfluxDBOrganization, Category: flags.MetricsCategory, } - - // Builder API flags - BuilderBlockValidationBlacklistSourceFilePath = &cli.StringFlag{ - Name: "builder.validation_blacklist", - Usage: "Path to file containing blacklisted addresses, json-encoded list of strings. Default assumes CWD is repo's root", - Value: "ofac_blacklist.json", - Category: flags.EthCategory, - } ) var ( @@ -1569,6 +1587,7 @@ func SetBuilderConfig(ctx *cli.Context, cfg *builder.Config) { cfg.GenesisValidatorsRoot = ctx.String(BuilderGenesisValidatorsRoot.Name) cfg.BeaconEndpoint = ctx.String(BuilderBeaconEndpoint.Name) cfg.RemoteRelayEndpoint = ctx.String(BuilderRemoteRelayEndpoint.Name) + cfg.SecondaryRemoteRelayEndpoints = strings.Split(ctx.String(BuilderSecondaryRemoteRelayEndpoints.Name), ",") cfg.ValidationBlocklist = ctx.String(BuilderBlockValidationBlacklistSourceFilePath.Name) } diff --git a/internal/flags/categories.go b/internal/flags/categories.go index c2db6c6c1d..779ec0ae5d 100644 --- a/internal/flags/categories.go +++ b/internal/flags/categories.go @@ -34,6 +34,7 @@ const ( LoggingCategory = "LOGGING AND DEBUGGING" MetricsCategory = "METRICS AND STATS" MiscCategory = "MISC" + BuilderCategory = "BUILDER" DeprecatedCategory = "ALIASED (deprecated)" )