Feat/state changes

cmd/node/CLI.md

@@ -73,6 +73,7 @@ GLOBAL OPTIONS:
    --operation-mode operation mode           String flag for specifying the desired operation mode(s) of the node, resulting in altering some configuration values accordingly. Possible values are: snapshotless-observer, full-archive, db-lookup-extension, historical-balances or `""` (empty). Multiple values can be separated via ,
    --repopulate-tokens-supplies              Boolean flag for repopulating the tokens supplies database. It will delete the current data, iterate over the entire trie and add he new obtained supplies
    --p2p-prometheus-metrics                  Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics
+   --state-accesses-types-to-collect value   String slice option for enabling collecting specified state accesses types. Can be (READ, WRITE)
    --help, -h                                show help
    --version, -v                             print the version
 

cmd/node/config/config.toml

@@ -355,6 +355,19 @@
         ShardIDProviderType = "BinarySplit"
         NumShards = 4
 
+[StateAccessesStorage]
+    [StateAccessesStorage.Cache]
+        Name = "StateAccessesStorage"
+        Capacity = 5000
+        Type = "SizeLRU"
+        SizeInBytes = 10485760 #10MB
+    [StateAccessesStorage.DB]
+        FilePath = "StateAccesses"
+        Type = "LvlDBSerial"
+        BatchDelaySeconds = 2
+        MaxBatchSize = 100
+        MaxOpenFiles = 10
+
 [PeerAccountsTrieStorage]
     [PeerAccountsTrieStorage.Cache]
         Name = "PeerAccountsTrieStorage"
@@ -696,6 +709,11 @@
     Enabled = false
     MaxSizeInBytes = 10485760 #10MB
 
+[StateAccessesCollectorConfig]
+    TypesToCollect = []          # These can be "READ","WRITE".
+    SaveToStorage = false
+    WithAccountChanges = false
+
 [BlockSizeThrottleConfig]
     MinSizeInBytes = 104857 # 104857 is 10% from 1MB
     MaxSizeInBytes = 943718 # 943718 is 90% from 1MB

cmd/node/flags.go

@@ -6,12 +6,13 @@ import (
 	"os"
 	"runtime"
 
+	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/urfave/cli"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/common/operationmodes"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/facade"
-	logger "github.com/multiversx/mx-chain-logger-go"
-	"github.com/urfave/cli"
 )
 
 var (
@@ -408,6 +409,13 @@ var (
 		Name:  "p2p-prometheus-metrics",
 		Usage: "Boolean option for enabling the /debug/metrics/prometheus route for p2p prometheus metrics",
 	}
+
+	// stateAccessesTypesToCollect defines a flag for collecting specified types of state accesses
+	// If enabled, it will override the configuration
+	stateAccessesTypesToCollect = cli.StringSliceFlag{
+		Name:  "state-accesses-types-to-collect",
+		Usage: "String slice option for enabling collecting specified state accesses types. Can be (READ, WRITE)",
+	}
 )
 
 func getFlags() []cli.Flag {
@@ -470,6 +478,7 @@ func getFlags() []cli.Flag {
 		operationMode,
 		repopulateTokensSupplies,
 		p2pPrometheusMetrics,
+		stateAccessesTypesToCollect,
 	}
 }
 

cmd/node/main.go

@@ -8,14 +8,15 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	"github.com/multiversx/mx-chain-core-go/core/check"
+	logger "github.com/multiversx/mx-chain-logger-go"
+	"github.com/multiversx/mx-chain-logger-go/file"
+	"github.com/urfave/cli"
+
 	"github.com/multiversx/mx-chain-go/cmd/node/factory"
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/config/overridableConfig"
 	"github.com/multiversx/mx-chain-go/node"
-	logger "github.com/multiversx/mx-chain-logger-go"
-	"github.com/multiversx/mx-chain-logger-go/file"
-	"github.com/urfave/cli"
 	// test point 1 for custom profiler
 )
 
@@ -260,6 +261,9 @@ func readConfigs(ctx *cli.Context, log logger.Logger) (*config.Configs, error) {
 	if ctx.IsSet(identityFlagName.Name) {
 		preferencesConfig.Preferences.Identity = ctx.GlobalString(identityFlagName.Name)
 	}
+	if ctx.IsSet(stateAccessesTypesToCollect.Name) {
+		generalConfig.StateAccessesCollectorConfig.TypesToCollect = ctx.GlobalStringSlice(stateAccessesTypesToCollect.Name)
+	}
 
 	return &config.Configs{
 		GeneralConfig:            generalConfig,

config/config.go

@@ -163,18 +163,20 @@ type Config struct {
 	SmartContractsStorageForSCQuery StorageConfig
 	TrieEpochRootHashStorage        StorageConfig
 	SmartContractsStorageSimulate   StorageConfig
+	StateAccessesStorage            StorageConfig
 
 	BootstrapStorage StorageConfig
 	MetaBlockStorage StorageConfig
 	ProofsStorage    StorageConfig
 
-	AccountsTrieStorage       StorageConfig
-	PeerAccountsTrieStorage   StorageConfig
-	EvictionWaitingList       EvictionWaitingListConfig
-	StateTriesConfig          StateTriesConfig
-	TrieStorageManagerConfig  TrieStorageManagerConfig
+	AccountsTrieStorage      StorageConfig
+	PeerAccountsTrieStorage  StorageConfig
+	EvictionWaitingList      EvictionWaitingListConfig
+	StateTriesConfig         StateTriesConfig
+	StateAccessesCollectorConfig StateAccessesCollectorConfig
+	TrieStorageManagerConfig TrieStorageManagerConfig
 	TrieLeavesRetrieverConfig TrieLeavesRetrieverConfig
-	BadBlocksCache            CacheConfig
+	BadBlocksCache           CacheConfig
 
 	TxBlockBodyDataPool         CacheConfig
 	PeerBlockBodyDataPool       CacheConfig
@@ -333,6 +335,13 @@ type StateTriesConfig struct {
 	StateStatisticsEnabled      bool
 }
 
+// StateAccessesCollectorConfig will hold information about state accesses collector
+type StateAccessesCollectorConfig struct {
+	TypesToCollect     []string
+	SaveToStorage      bool
+	WithAccountChanges bool
+}
+
 // TrieStorageManagerConfig will hold config information about trie storage manager
 type TrieStorageManagerConfig struct {
 	PruningBufferLen      uint32

dataRetriever/unitType.go

@@ -51,6 +51,8 @@ const (
 	ScheduledSCRsUnit UnitType = 22
 	// ProofsUnit is the header proofs unit identifier
 	ProofsUnit UnitType = 23
+	// StateAccessesUnit is the state accesses storage unit identifier
+	StateAccessesUnit UnitType = 24
 
 	// ShardHdrNonceHashDataUnit is the header nonce-hash pair data unit identifier
 	//TODO: Add only unit types lower than 100
@@ -114,6 +116,8 @@ func (ut UnitType) String() string {
 		return "ScheduledSCRsUnit"
 	case ProofsUnit:
 		return "ProofsUnit"
+	case StateAccessesUnit:
+		return "StateAccessesUnit"
 	}
 
 	if ut < ShardHdrNonceHashDataUnit {

epochStart/bootstrap/disabled/disabledAccountsAdapter.go

@@ -128,6 +128,10 @@ func (a *accountsAdapter) GetStackDebugFirstEntry() []byte {
 	return nil
 }
 
+// SetTxHashForLatestStateAccesses -
+func (a *accountsAdapter) SetTxHashForLatestStateAccesses(_ []byte) {
+}
+
 // Close -
 func (a *accountsAdapter) Close() error {
 	return nil

epochStart/metachain/baseRewards_test.go

@@ -14,6 +14,10 @@ import (
 	"github.com/multiversx/mx-chain-core-go/data/rewardTx"
 	"github.com/multiversx/mx-chain-core-go/hashing/sha256"
 	"github.com/multiversx/mx-chain-core-go/marshal"
+	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+
 	"github.com/multiversx/mx-chain-go/epochStart"
 	"github.com/multiversx/mx-chain-go/epochStart/mock"
 	"github.com/multiversx/mx-chain-go/process"
@@ -27,9 +31,6 @@ import (
 	stateMock "github.com/multiversx/mx-chain-go/testscommon/state"
 	"github.com/multiversx/mx-chain-go/testscommon/storage"
 	"github.com/multiversx/mx-chain-go/trie"
-	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestBaseRewardsCreator_NilShardCoordinator(t *testing.T) {
@@ -1160,9 +1161,10 @@ func getBaseRewardsArguments() BaseRewardsCreatorArgs {
 
 	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storage.GetStorageManagerOptions())
 	argsAccCreator := factory.ArgsAccountCreator{
-		Hasher:              hasher,
-		Marshaller:          marshalizer,
-		EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		Hasher:                 hasher,
+		Marshaller:             marshalizer,
+		EnableEpochsHandler:    &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		StateAccessesCollector: &stateMock.StateAccessesCollectorStub{},
 	}
 	accCreator, _ := factory.NewAccountCreator(argsAccCreator)
 	enableEpochsHandler := &enableEpochsHandlerMock.EnableEpochsHandlerStub{}

epochStart/metachain/systemSCs_test.go

@@ -756,13 +756,14 @@ func createAccountsDB(
 	spm, _ := storagePruningManager.NewStoragePruningManager(ewl, 10)
 
 	args := state.ArgsAccountsDB{
-		Trie:                  tr,
-		Hasher:                hasher,
-		Marshaller:            marshaller,
-		AccountFactory:        accountFactory,
-		StoragePruningManager: spm,
-		AddressConverter:      &testscommon.PubkeyConverterMock{},
-		SnapshotsManager:      disabledState.NewDisabledSnapshotsManager(),
+		Trie:                   tr,
+		Hasher:                 hasher,
+		Marshaller:             marshaller,
+		AccountFactory:         accountFactory,
+		StoragePruningManager:  spm,
+		AddressConverter:       &testscommon.PubkeyConverterMock{},
+		SnapshotsManager:       disabledState.NewDisabledSnapshotsManager(),
+		StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
 	}
 	adb, _ := state.NewAccountsDB(args)
 	return adb
@@ -778,9 +779,10 @@ func createFullArgumentsForSystemSCProcessing(enableEpochsConfig config.EnableEp
 
 	trieFactoryManager, _ := trie.CreateTrieStorageManager(storageManagerArgs, storageMock.GetStorageManagerOptions())
 	argsAccCreator := factory.ArgsAccountCreator{
-		Hasher:              hasher,
-		Marshaller:          marshalizer,
-		EnableEpochsHandler: &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		Hasher:                 hasher,
+		Marshaller:             marshalizer,
+		EnableEpochsHandler:    &enableEpochsHandlerMock.EnableEpochsHandlerStub{},
+		StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
 	}
 	accCreator, _ := factory.NewAccountCreator(argsAccCreator)
 	peerAccCreator := factory.NewPeerAccountCreator()

factory/api/apiResolverFactory.go

@@ -550,9 +550,10 @@ func createShardVmContainerFactory(args scQueryElementArgs, argsHook hooks.ArgBl
 
 func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.ChainHandler) (state.AccountsAdapterAPI, common.StorageManager, error) {
 	argsAccCreator := factoryState.ArgsAccountCreator{
-		Hasher:              args.coreComponents.Hasher(),
-		Marshaller:          args.coreComponents.InternalMarshalizer(),
-		EnableEpochsHandler: args.coreComponents.EnableEpochsHandler(),
+		Hasher:                 args.coreComponents.Hasher(),
+		Marshaller:             args.coreComponents.InternalMarshalizer(),
+		EnableEpochsHandler:    args.coreComponents.EnableEpochsHandler(),
+		StateAccessesCollector: args.stateComponents.StateAccessesCollector(),
 	}
 	accountFactory, err := factoryState.NewAccountCreator(argsAccCreator)
 	if err != nil {
@@ -596,13 +597,14 @@ func createNewAccountsAdapterApi(args scQueryElementArgs, chainHandler data.Chai
 	}
 
 	argsAPIAccountsDB := state.ArgsAccountsDB{
-		Trie:                  merkleTrie,
-		Hasher:                args.coreComponents.Hasher(),
-		Marshaller:            args.coreComponents.InternalMarshalizer(),
-		AccountFactory:        accountFactory,
-		StoragePruningManager: storagePruning,
-		AddressConverter:      args.coreComponents.AddressPubKeyConverter(),
-		SnapshotsManager:      disabledState.NewDisabledSnapshotsManager(),
+		Trie:                   merkleTrie,
+		Hasher:                 args.coreComponents.Hasher(),
+		Marshaller:             args.coreComponents.InternalMarshalizer(),
+		AccountFactory:         accountFactory,
+		StoragePruningManager:  storagePruning,
+		AddressConverter:       args.coreComponents.AddressPubKeyConverter(),
+		SnapshotsManager:       disabledState.NewDisabledSnapshotsManager(),
+		StateAccessesCollector: disabledState.NewDisabledStateAccessesCollector(),
 	}
 
 	provider, err := blockInfoProviders.NewCurrentBlockInfo(chainHandler)

factory/api/apiResolverFactory_test.go

@@ -19,6 +19,7 @@ import (
 	"github.com/multiversx/mx-chain-go/process"
 	"github.com/multiversx/mx-chain-go/process/sync/disabled"
 	"github.com/multiversx/mx-chain-go/state"
+	stateDisabled "github.com/multiversx/mx-chain-go/state/disabled"
 	"github.com/multiversx/mx-chain-go/testscommon"
 	componentsMock "github.com/multiversx/mx-chain-go/testscommon/components"
 	"github.com/multiversx/mx-chain-go/testscommon/dataRetriever"
@@ -344,6 +345,9 @@ func createMockSCQueryElementArgs() api.SCQueryElementArgs {
 			PeerAccountsCalled: func() state.AccountsAdapter {
 				return &stateMocks.AccountsStub{}
 			},
+			StateAccessesCollectorCalled: func() state.StateAccessesCollector {
+				return stateDisabled.NewDisabledStateAccessesCollector()
+			},
 		},
 		StatusCoreComponents: &factory.StatusCoreComponentsStub{
 			AppStatusHandlerCalled: func() core.AppStatusHandler {

factory/interface.go

@@ -344,6 +344,7 @@ type StateComponentsHolder interface {
 	TrieStorageManagers() map[string]common.StorageManager
 	MissingTrieNodesNotifier() common.MissingTrieNodesNotifier
 	TrieLeavesRetriever() common.TrieLeavesRetriever
+	StateAccessesCollector() state.StateAccessesCollector
 	Close() error
 	IsInterfaceNil() bool
 }

factory/mock/stateComponentsHolderStub.go

@@ -15,6 +15,7 @@ type StateComponentsHolderStub struct {
 	TrieStorageManagersCalled      func() map[string]common.StorageManager
 	MissingTrieNodesNotifierCalled func() common.MissingTrieNodesNotifier
 	TrieLeavesRetrieverCalled      func() common.TrieLeavesRetriever
+	StateAccessesCollectorCalled   func() state.StateAccessesCollector
 }
 
 // PeerAccounts -
@@ -88,6 +89,15 @@ func (s *StateComponentsHolderStub) TrieLeavesRetriever() common.TrieLeavesRetri
 	return nil
 }
 
+// StateAccessesCollector -
+func (s *StateComponentsHolderStub) StateAccessesCollector() state.StateAccessesCollector {
+	if s.StateAccessesCollectorCalled != nil {
+		return s.StateAccessesCollectorCalled()
+	}
+
+	return nil
+}
+
 // Close -
 func (s *StateComponentsHolderStub) Close() error {
 	return nil

factory/processing/blockProcessorCreator.go

@@ -6,6 +6,10 @@ import (
 
 	"github.com/multiversx/mx-chain-core-go/core"
 	dataBlock "github.com/multiversx/mx-chain-core-go/data/block"
+	logger "github.com/multiversx/mx-chain-logger-go"
+	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
+	"github.com/multiversx/mx-chain-vm-common-go/parsers"
+
 	"github.com/multiversx/mx-chain-go/common"
 	"github.com/multiversx/mx-chain-go/config"
 	"github.com/multiversx/mx-chain-go/dataRetriever"
@@ -42,9 +46,6 @@ import (
 	"github.com/multiversx/mx-chain-go/state/syncer"
 	"github.com/multiversx/mx-chain-go/storage/txcache"
 	"github.com/multiversx/mx-chain-go/vm"
-	logger "github.com/multiversx/mx-chain-logger-go"
-	vmcommon "github.com/multiversx/mx-chain-vm-common-go"
-	"github.com/multiversx/mx-chain-vm-common-go/parsers"
 )
 
 type blockProcessorAndVmFactories struct {
@@ -274,6 +275,8 @@ func (pcf *processComponentsFactory) newShardBlockProcessor(
 		pcf.state.AccountsAdapter(),
 		pcf.coreData.AddressPubKeyConverter(),
 		pcf.bootstrapComponents.ShardCoordinator(),
+		pcf.coreData.InternalMarshalizer(),
+		pcf.coreData.Hasher(),
 	)
 	if err != nil {
 		return nil, err
@@ -442,6 +445,7 @@ func (pcf *processComponentsFactory) newShardBlockProcessor(
 		BlockProcessingCutoffHandler: blockProcessingCutoffHandler,
 		ManagedPeersHolder:           pcf.crypto.ManagedPeersHolder(),
 		SentSignaturesTracker:        sentSignaturesTracker,
+		StateAccessesCollector:       pcf.state.StateAccessesCollector(),
 	}
 	arguments := block.ArgShardProcessor{
 		ArgBaseProcessor: argumentsBaseProcessor,
@@ -879,6 +883,7 @@ func (pcf *processComponentsFactory) newMetaBlockProcessor(
 		BlockProcessingCutoffHandler: blockProcessingCutoffhandler,
 		ManagedPeersHolder:           pcf.crypto.ManagedPeersHolder(),
 		SentSignaturesTracker:        sentSignaturesTracker,
+		StateAccessesCollector:       pcf.state.StateAccessesCollector(),
 	}
 
 	esdtOwnerAddress, err := pcf.coreData.AddressPubKeyConverter().Decode(pcf.systemSCConfig.ESDTSystemSCConfig.OwnerAddress)
@@ -1024,7 +1029,7 @@ func (pcf *processComponentsFactory) createOutportDataProvider(
 	return factoryOutportProvider.CreateOutportDataProvider(factoryOutportProvider.ArgOutportDataProviderFactory{
 		HasDrivers:             pcf.statusComponents.OutportHandler().HasDrivers(),
 		AddressConverter:       pcf.coreData.AddressPubKeyConverter(),
-		AccountsDB:             pcf.state.AccountsAdapter(),
+		AccountsDB:             pcf.state.AccountsAdapterAPI(),
 		Marshaller:             pcf.coreData.InternalMarshalizer(),
 		EsdtDataStorageHandler: pcf.esdtNftStorage,
 		TransactionsStorer:     txsStorer,
@@ -1039,6 +1044,7 @@ func (pcf *processComponentsFactory) createOutportDataProvider(
 		EnableEpochsHandler:    pcf.coreData.EnableEpochsHandler(),
 		ExecutionOrderGetter:   pcf.txExecutionOrderHandler,
 		ProofsPool:             pcf.data.Datapool().Proofs(),
+		StateAccessesCollector: pcf.state.StateAccessesCollector(),
 	})
 }