Skip to content

EN-13432: Nodes shuffler refactoring #4799

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Dec 21, 2022
Merged

EN-13432: Nodes shuffler refactoring #4799

Show file tree
Hide file tree
Changes from 2 commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
156b5ae
EN-13432: shuffler refactoring
bogdan-rosianu Dec 13, 2022
3bb1758
Merge branch 'feat/consensus-size-changes' into EN-13432-shuffler-ref…
bogdan-rosianu Dec 14, 2022
1fad090
EN-13432: fixes after review:
bogdan-rosianu Dec 16, 2022
7efe443
EN-13432: stub and mock cleanup
bogdan-rosianu Dec 21, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 4 additions & 7 deletions factory/core/coreComponents.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,13 +305,10 @@ func (ccf *coreComponentsFactory) Create() (*coreComponents, error) {
}

argsNodesShuffler := &nodesCoordinator.NodesShufflerArgs{
NodesShard: genesisNodesConfig.MinNumberOfShardNodes(),
NodesMeta: genesisNodesConfig.MinNumberOfMetaNodes(),
Hysteresis: genesisNodesConfig.GetHysteresis(),
Adaptivity: genesisNodesConfig.GetAdaptivity(),
ShuffleBetweenShards: true,
MaxNodesEnableConfig: ccf.epochConfig.EnableEpochs.MaxNodesChangeEnableEpoch,
EnableEpochsHandler: enableEpochsHandler,
ChainParametersHandler: chainParametersHandler,
ShuffleBetweenShards: true,
MaxNodesEnableConfig: ccf.epochConfig.EnableEpochs.MaxNodesChangeEnableEpoch,
EnableEpochsHandler: enableEpochsHandler,
}

nodesShuffler, err := nodesCoordinator.NewHashValidatorsShuffler(argsNodesShuffler)
Expand Down
30 changes: 22 additions & 8 deletions integrationTests/nodesCoordinatorFactory.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,14 @@ import (

"github.com/ElrondNetwork/elrond-go-core/data/endProcess"
"github.com/ElrondNetwork/elrond-go-core/hashing"
"github.com/ElrondNetwork/elrond-go/config"
"github.com/ElrondNetwork/elrond-go/integrationTests/mock"
"github.com/ElrondNetwork/elrond-go/sharding"
"github.com/ElrondNetwork/elrond-go/sharding/nodesCoordinator"
"github.com/ElrondNetwork/elrond-go/storage"
"github.com/ElrondNetwork/elrond-go/testscommon"
"github.com/ElrondNetwork/elrond-go/testscommon/nodeTypeProviderMock"
"github.com/ElrondNetwork/elrond-go/testscommon/shardingmock"
vic "github.com/ElrondNetwork/elrond-go/testscommon/validatorInfoCacher"
)

Expand Down Expand Up @@ -43,10 +45,16 @@ func (tpn *IndexHashedNodesCoordinatorFactory) CreateNodesCoordinator(arg ArgInd
pubKeyBytes, _ := keys.Pk.ToByteArray()

nodeShufflerArgs := &nodesCoordinator.NodesShufflerArgs{
NodesShard: uint32(arg.nodesPerShard),
NodesMeta: uint32(arg.nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
ChainParametersHandler: &shardingmock.ChainParametersHandlerStub{
CurrentChainParametersCalled: func() config.ChainParametersByEpochConfig {
return config.ChainParametersByEpochConfig{
ShardMinNumNodes: uint32(arg.nodesPerShard),
MetachainMinNumNodes: uint32(arg.nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
}
},
},
ShuffleBetweenShards: shuffleBetweenShards,
MaxNodesEnableConfig: nil,
EnableEpochsHandler: &testscommon.EnableEpochsHandlerStub{},
Expand Down Expand Up @@ -97,10 +105,16 @@ func (ihncrf *IndexHashedNodesCoordinatorWithRaterFactory) CreateNodesCoordinato
pubKeyBytes, _ := keys.Pk.ToByteArray()

shufflerArgs := &nodesCoordinator.NodesShufflerArgs{
NodesShard: uint32(arg.nodesPerShard),
NodesMeta: uint32(arg.nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
ChainParametersHandler: &shardingmock.ChainParametersHandlerStub{
CurrentChainParametersCalled: func() config.ChainParametersByEpochConfig {
return config.ChainParametersByEpochConfig{
ShardMinNumNodes: uint32(arg.nodesPerShard),
MetachainMinNumNodes: uint32(arg.nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
}
},
},
ShuffleBetweenShards: shuffleBetweenShards,
MaxNodesEnableConfig: nil,
EnableEpochsHandler: &testscommon.EnableEpochsHandlerStub{
Expand Down
15 changes: 11 additions & 4 deletions integrationTests/testProcessorNodeWithMultisigner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"github.com/ElrondNetwork/elrond-go/testscommon/cryptoMocks"
"github.com/ElrondNetwork/elrond-go/testscommon/nodeTypeProviderMock"
"github.com/ElrondNetwork/elrond-go/testscommon/shardingMocks"
"github.com/ElrondNetwork/elrond-go/testscommon/shardingmock"
vic "github.com/ElrondNetwork/elrond-go/testscommon/validatorInfoCacher"
)

Expand Down Expand Up @@ -392,10 +393,16 @@ func CreateNodesWithNodesCoordinatorAndHeaderSigVerifier(
nodesMap := make(map[uint32][]*TestProcessorNode)

shufflerArgs := &nodesCoordinator.NodesShufflerArgs{
NodesShard: uint32(nodesPerShard),
NodesMeta: uint32(nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
ChainParametersHandler: &shardingmock.ChainParametersHandlerStub{
CurrentChainParametersCalled: func() config.ChainParametersByEpochConfig {
return config.ChainParametersByEpochConfig{
ShardMinNumNodes: uint32(nodesPerShard),
MetachainMinNumNodes: uint32(nbMetaNodes),
Hysteresis: hysteresis,
Adaptivity: adaptivity,
}
},
},
ShuffleBetweenShards: shuffleBetweenShards,
MaxNodesEnableConfig: nil,
EnableEpochsHandler: &testscommon.EnableEpochsHandlerStub{},
Expand Down
3 changes: 3 additions & 0 deletions sharding/nodesCoordinator/errors.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,3 +108,6 @@ var ErrNilEnableEpochsHandler = errors.New("nil enable epochs handler")

// ErrNilValidatorInfoCacher signals that a nil value for the validator info cacher has been provided
var ErrNilValidatorInfoCacher = errors.New("validator info cacher is nil")

// ErrNilChainParametersHandler signals that a nil chain parameters handler has been provided
var ErrNilChainParametersHandler = errors.New("nil chain parameters handler")
82 changes: 33 additions & 49 deletions sharding/nodesCoordinator/hashValidatorShuffler.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,10 @@ var _ NodesShuffler = (*randHashShuffler)(nil)

// NodesShufflerArgs defines the arguments required to create a nodes shuffler
type NodesShufflerArgs struct {
NodesShard uint32
NodesMeta uint32
Hysteresis float32
Adaptivity bool
ShuffleBetweenShards bool
MaxNodesEnableConfig []config.MaxNodesChangeConfig
EnableEpochsHandler common.EnableEpochsHandler
ShuffleBetweenShards bool
MaxNodesEnableConfig []config.MaxNodesChangeConfig
EnableEpochsHandler common.EnableEpochsHandler
ChainParametersHandler ChainParametersHandler
Copy link
Contributor

@AdoAdoAdo AdoAdoAdo Dec 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

check also if the notify order for chain parameters handler.
it needs to at least be before nodes coordinator.

Copy link
Contributor Author

@bogdan-rosianu bogdan-rosianu Dec 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

refactored - so that it uses the epoch notifier with ordering

}

type shuffleNodesArg struct {
Expand All @@ -49,11 +46,9 @@ type randHashShuffler struct {
// when reinitialization of node in new shard is implemented
shuffleBetweenShards bool

adaptivity bool
nodesShard uint32
nodesMeta uint32
shardHysteresis uint32
metaHysteresis uint32
currentChainParameters config.ChainParametersByEpochConfig
//shardHysteresis uint32
Copy link
Contributor

@AdoAdoAdo AdoAdoAdo Dec 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

remove commented out code

Copy link
Contributor Author

@bogdan-rosianu bogdan-rosianu Dec 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

removed

//metaHysteresis uint32
activeNodesConfig config.MaxNodesChangeConfig
availableNodesConfigs []config.MaxNodesChangeConfig
mutShufflerParams sync.RWMutex
Expand All @@ -72,7 +67,9 @@ func NewHashValidatorsShuffler(args *NodesShufflerArgs) (*randHashShuffler, erro
if check.IfNil(args.EnableEpochsHandler) {
return nil, ErrNilEnableEpochsHandler
}

if check.IfNil(args.ChainParametersHandler) {
return nil, ErrNilChainParametersHandler
}
var configs []config.MaxNodesChangeConfig

log.Debug("hashValidatorShuffler: enable epoch for max nodes change", "epoch", args.MaxNodesEnableConfig)
Expand All @@ -81,15 +78,15 @@ func NewHashValidatorsShuffler(args *NodesShufflerArgs) (*randHashShuffler, erro
copy(configs, args.MaxNodesEnableConfig)
}

currentChainParameters := args.ChainParametersHandler.CurrentChainParameters()
log.Debug("Shuffler created", "shuffleBetweenShards", args.ShuffleBetweenShards)
rxs := &randHashShuffler{
shuffleBetweenShards: args.ShuffleBetweenShards,
availableNodesConfigs: configs,
enableEpochsHandler: args.EnableEpochsHandler,
currentChainParameters: currentChainParameters,
shuffleBetweenShards: args.ShuffleBetweenShards,
availableNodesConfigs: configs,
enableEpochsHandler: args.EnableEpochsHandler,
}

rxs.UpdateParams(args.NodesShard, args.NodesMeta, args.Hysteresis, args.Adaptivity)

if rxs.shuffleBetweenShards {
rxs.validatorDistributor = &CrossShardValidatorDistributor{}
} else {
Expand All @@ -101,27 +98,6 @@ func NewHashValidatorsShuffler(args *NodesShufflerArgs) (*randHashShuffler, erro
return rxs, nil
}

// UpdateParams updates the shuffler parameters
// Should be called when new params are agreed through governance
func (rhs *randHashShuffler) UpdateParams(
nodesShard uint32,
nodesMeta uint32,
hysteresis float32,
adaptivity bool,
) {
// TODO: are there constraints we want to enforce? e.g min/max hysteresis
shardHysteresis := uint32(float32(nodesShard) * hysteresis)
metaHysteresis := uint32(float32(nodesMeta) * hysteresis)

rhs.mutShufflerParams.Lock()
rhs.shardHysteresis = shardHysteresis
rhs.metaHysteresis = metaHysteresis
rhs.nodesShard = nodesShard
rhs.nodesMeta = nodesMeta
rhs.adaptivity = adaptivity
rhs.mutShufflerParams.Unlock()
}

// UpdateNodeLists shuffles the nodes and returns the lists with the new nodes configuration
// The function needs to ensure that:
// 1. Old eligible nodes list will have up to shuffleOutThreshold percent nodes shuffled out from each shard
Expand All @@ -143,7 +119,7 @@ func (rhs *randHashShuffler) UpdateNodeLists(args ArgsUpdateNodes) (*ResUpdateNo
eligibleAfterReshard := copyValidatorMap(args.Eligible)
waitingAfterReshard := copyValidatorMap(args.Waiting)

args.AdditionalLeaving = removeDupplicates(args.UnStakeLeaving, args.AdditionalLeaving)
args.AdditionalLeaving = removeDuplicates(args.UnStakeLeaving, args.AdditionalLeaving)
totalLeavingNum := len(args.AdditionalLeaving) + len(args.UnStakeLeaving)

newNbShards := rhs.computeNewShards(
Expand All @@ -155,10 +131,10 @@ func (rhs *randHashShuffler) UpdateNodeLists(args ArgsUpdateNodes) (*ResUpdateNo
)

rhs.mutShufflerParams.RLock()
canSplit := rhs.adaptivity && newNbShards > args.NbShards
canMerge := rhs.adaptivity && newNbShards < args.NbShards
nodesPerShard := rhs.nodesShard
nodesMeta := rhs.nodesMeta
canSplit := rhs.currentChainParameters.Adaptivity && newNbShards > args.NbShards
canMerge := rhs.currentChainParameters.Adaptivity && newNbShards < args.NbShards
nodesPerShard := rhs.currentChainParameters.ShardMinNumNodes
nodesMeta := rhs.currentChainParameters.MetachainMinNumNodes
rhs.mutShufflerParams.RUnlock()

if canSplit {
Expand All @@ -185,7 +161,7 @@ func (rhs *randHashShuffler) UpdateNodeLists(args ArgsUpdateNodes) (*ResUpdateNo
})
}

func removeDupplicates(unstake []Validator, additionalLeaving []Validator) []Validator {
func removeDuplicates(unstake []Validator, additionalLeaving []Validator) []Validator {
additionalCopy := make([]Validator, 0, len(additionalLeaving))
additionalCopy = append(additionalCopy, additionalLeaving...)

Expand Down Expand Up @@ -452,10 +428,10 @@ func (rhs *randHashShuffler) computeNewShards(
nodesNewEpoch := uint32(nbEligible + nbWaiting + numNewNodes - numLeavingNodes)

rhs.mutShufflerParams.RLock()
maxNodesMeta := rhs.nodesMeta + rhs.metaHysteresis
maxNodesShard := rhs.nodesShard + rhs.shardHysteresis
maxNodesMeta := rhs.currentChainParameters.MetachainMinNumNodes + rhs.metaHysteresis()
maxNodesShard := rhs.currentChainParameters.ShardMinNumNodes + rhs.shardHysteresis()
nodesForSplit := (nbShards+1)*maxNodesShard + maxNodesMeta
nodesForMerge := nbShards*rhs.nodesShard + rhs.nodesMeta
nodesForMerge := nbShards*rhs.currentChainParameters.ShardMinNumNodes + rhs.currentChainParameters.MetachainMinNumNodes
rhs.mutShufflerParams.RUnlock()

nbShardsNew := nbShards
Expand All @@ -473,6 +449,14 @@ func (rhs *randHashShuffler) computeNewShards(
return nbShardsNew
}

func (rhs *randHashShuffler) metaHysteresis() uint32 {
return uint32(rhs.currentChainParameters.Hysteresis * float32(rhs.currentChainParameters.MetachainMinNumNodes))
}

func (rhs *randHashShuffler) shardHysteresis() uint32 {
return uint32(rhs.currentChainParameters.Hysteresis * float32(rhs.currentChainParameters.ShardMinNumNodes))
}

// shuffleOutNodes shuffles the list of eligible validators in each shard and returns the map of shuffled out
// validators
func shuffleOutNodes(
Expand Down Expand Up @@ -761,7 +745,7 @@ func sortKeys(nodes map[uint32][]Validator) []uint32 {
func (rhs *randHashShuffler) UpdateShufflerConfig(epoch uint32) {
rhs.mutShufflerParams.Lock()
defer rhs.mutShufflerParams.Unlock()
rhs.activeNodesConfig.NodesToShufflePerShard = rhs.nodesShard
rhs.activeNodesConfig.NodesToShufflePerShard = rhs.currentChainParameters.ShardMinNumNodes
for _, maxNodesConfig := range rhs.availableNodesConfigs {
if epoch >= maxNodesConfig.EpochEnable {
rhs.activeNodesConfig = maxNodesConfig
Expand Down
Loading