Skip to content

feat(dot/rpc): implement RPC method state_queryStorageAt #3185

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
edwardmack wants to merge 7 commits into from
Closed

feat(dot/rpc): implement RPC method state_queryStorageAt #3185

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
d1e9475
refactor(runtimes): remove deprecated node runtimes (#3177)
dimartiro Apr 4, 2023
af38364
fix(cmd/gossamer): embed default toml config files (#3091)
edwardmack Apr 5, 2023
0c3214d
implement QueryStorageAt
edwardmack Mar 30, 2023
f7c95f6
add unit tests for QueryStorageAt
edwardmack Mar 30, 2023
683adc4
rename variable
edwardmack Mar 30, 2023
39b2b5c
address PR comments, handle empty [] vs unset []
edwardmack Apr 4, 2023
49737fa
address PR comments, clean-up comments
edwardmack Apr 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions chain/resources.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
// Copyright 2023 ChainSafe Systems (ON)
// SPDX-License-Identifier: LGPL-3.0-only
package chain

import "embed"

// DefaultConfigTomlFiles is the embedded file system containing the default toml configurations.
//
//go:embed */*.toml
var DefaultConfigTomlFiles embed.FS
20 changes: 10 additions & 10 deletions cmd/gossamer/config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,18 +26,18 @@ import (
var (
// DefaultCfg is the default configuration for the node.
DefaultCfg = dot.PolkadotConfig
defaultKusamaConfigPath = "./chain/kusama/config.toml"
defaultPolkadotConfigPath = "./chain/polkadot/config.toml"
defaultWestendDevConfigPath = "./chain/westend-dev/config.toml"
defaultWestendConfigPath = "./chain/westend/config.toml"
defaultKusamaConfigPath = "kusama/config.toml"
defaultPolkadotConfigPath = "polkadot/config.toml"
defaultWestendDevConfigPath = "westend-dev/config.toml"
defaultWestendConfigPath = "westend/config.toml"
)

// loadConfigFile loads a default config file if --chain is specified, a specific
// config if --config is specified, or the default gossamer config otherwise.
func loadConfigFile(ctx *cli.Context, cfg *ctoml.Config) (err error) {
cfgPath := ctx.String(ConfigFlag.Name)
if cfgPath == "" {
return loadConfig(cfg, defaultPolkadotConfigPath)
return loadConfigFromResource(cfg, defaultPolkadotConfigPath)
}

logger.Info("loading toml configuration from " + cfgPath + "...")
Expand All @@ -48,7 +48,7 @@ func loadConfigFile(ctx *cli.Context, cfg *ctoml.Config) (err error) {
"overwriting default configuration with id " + cfg.Global.ID +
" with toml configuration values from " + cfgPath)
}
return loadConfig(cfg, cfgPath)
return loadConfigFromFile(cfg, cfgPath)
}

func setupConfigFromChain(ctx *cli.Context) (*ctoml.Config, *dot.Config, error) {
Expand All @@ -68,22 +68,22 @@ func setupConfigFromChain(ctx *cli.Context) (*ctoml.Config, *dot.Config, error)
logger.Info("loading toml configuration from " + defaultKusamaConfigPath + "...")
tomlCfg = &ctoml.Config{}
cfg = dot.KusamaConfig()
err = loadConfig(tomlCfg, defaultKusamaConfigPath)
err = loadConfigFromResource(tomlCfg, defaultKusamaConfigPath)
case "polkadot":
logger.Info("loading toml configuration from " + defaultPolkadotConfigPath + "...")
tomlCfg = &ctoml.Config{}
cfg = dot.PolkadotConfig()
err = loadConfig(tomlCfg, defaultPolkadotConfigPath)
err = loadConfigFromResource(tomlCfg, defaultPolkadotConfigPath)
case "westend-dev":
logger.Info("loading toml configuration from " + defaultWestendDevConfigPath + "...")
tomlCfg = &ctoml.Config{}
cfg = dot.WestendDevConfig()
err = loadConfig(tomlCfg, defaultWestendDevConfigPath)
err = loadConfigFromResource(tomlCfg, defaultWestendDevConfigPath)
case "westend":
logger.Info("loading toml configuration from " + defaultWestendConfigPath + "...")
tomlCfg = &ctoml.Config{}
cfg = dot.WestendConfig()
err = loadConfig(tomlCfg, defaultWestendConfigPath)
err = loadConfigFromResource(tomlCfg, defaultWestendConfigPath)
default:
logger.Info("loading chain config from " + id + "...")
fileInfo, err := os.Stat(id)
Expand Down
2 changes: 1 addition & 1 deletion cmd/gossamer/config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -845,7 +845,7 @@ func TestGlobalNodeName_WhenNodeAlreadyHasStoredName(t *testing.T) {
cfg := newTestConfig(t, westendDevConfig)
cfg.Global.Name = globalName

runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.NODE_RUNTIME)
runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.WESTEND_RUNTIME_v0929)
require.NoError(t, err)
runtimeData, err := os.ReadFile(runtimeFilePath)
require.NoError(t, err)
Expand Down
3 changes: 1 addition & 2 deletions cmd/gossamer/export_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@ import (
"testing"

"github.com/ChainSafe/gossamer/dot"

ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
"github.com/ChainSafe/gossamer/internal/log"
"github.com/stretchr/testify/require"
Expand Down Expand Up @@ -165,7 +164,7 @@ func TestExportCommand(t *testing.T) {
config := ctx.String(ConfigFlag.Name)

cfg := new(ctoml.Config)
err = loadConfig(cfg, config)
err = loadConfigFromFile(cfg, config)
require.NoError(t, err)

require.Equal(t, dotConfigToToml(c.expected), cfg)
Expand Down
11 changes: 5 additions & 6 deletions cmd/gossamer/main_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,8 @@ func runTestGossamer(t *testing.T, args ...string) *testgossamer {
return tt
}

var testWestendDevConfigPath string

func TestMain(m *testing.M) {
if reexec.Init() {
return
Expand All @@ -202,10 +204,7 @@ func TestMain(m *testing.M) {
panic(err)
}

defaultKusamaConfigPath = filepath.Join(rootPath, "./chain/kusama/config.toml")
defaultPolkadotConfigPath = filepath.Join(rootPath, "./chain/polkadot/config.toml")
defaultWestendDevConfigPath = filepath.Join(rootPath, "./chain/westend-dev/config.toml")

testWestendDevConfigPath = filepath.Join(rootPath, "./chain/westend-dev/config.toml")
os.Exit(m.Run())
}

Expand Down Expand Up @@ -234,7 +233,7 @@ func TestInitCommand_RenameNodeWhenCalled(t *testing.T) {
"--basepath", tempDir,
"--genesis", genesisPath,
"--name", nodeName,
"--config", defaultWestendDevConfigPath,
"--config", testWestendDevConfigPath,
"--force",
)

Expand All @@ -249,7 +248,7 @@ func TestInitCommand_RenameNodeWhenCalled(t *testing.T) {
"init",
"--basepath", tempDir,
"--genesis", genesisPath,
"--config", defaultWestendDevConfigPath,
"--config", testWestendDevConfigPath,
"--force",
)

Expand Down
20 changes: 15 additions & 5 deletions cmd/gossamer/toml_config.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,25 +11,35 @@ import (
"reflect"
"unicode"

"github.com/naoina/toml"

"github.com/ChainSafe/gossamer/chain"
ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
"github.com/naoina/toml"
)

// loadConfig loads the values from the toml configuration file into the provided configuration
func loadConfig(cfg *ctoml.Config, fp string) error {
func loadConfigFromResource(cfg *ctoml.Config, resourcePath string) error {
file, err := chain.DefaultConfigTomlFiles.Open(resourcePath)
if err != nil {
return fmt.Errorf("opening toml configuration file: %w", err)
}
return loadConfig(cfg, file)
}

func loadConfigFromFile(cfg *ctoml.Config, fp string) error {
fp, err := filepath.Abs(fp)
if err != nil {
logger.Errorf("failed to create absolute path for toml configuration file: %s", err)
return err
}

file, err := os.Open(filepath.Clean(fp))
if err != nil {
logger.Errorf("failed to open toml configuration file: %s", err)
return err
}
return loadConfig(cfg, file)
}

// loadConfig loads the values from the toml configuration file into the provided configuration
func loadConfig(cfg *ctoml.Config, file fs.File) (err error) {
var tomlSettings = toml.Config{
NormFieldName: func(rt reflect.Type, key string) string {
return key
Expand Down
6 changes: 3 additions & 3 deletions cmd/gossamer/toml_config_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func TestLoadConfig(t *testing.T) {
err := dot.InitNode(cfg)
require.NoError(t, err)

err = loadConfig(dotConfigToToml(cfg), cfgFile)
err = loadConfigFromFile(dotConfigToToml(cfg), cfgFile)
require.NoError(t, err)
}

Expand All @@ -43,7 +43,7 @@ func TestLoadConfigWestendDev(t *testing.T) {
projectRootPath := utils.GetProjectRootPathTest(t)
configPath := filepath.Join(projectRootPath, "./chain/westend-dev/config.toml")

err = loadConfig(dotConfigToToml(cfg), configPath)
err = loadConfigFromFile(dotConfigToToml(cfg), configPath)
require.NoError(t, err)
}

Expand All @@ -60,6 +60,6 @@ func TestLoadConfigKusama(t *testing.T) {
projectRootPath := utils.GetProjectRootPathTest(t)
kusamaConfigPath := filepath.Join(projectRootPath, "./chain/kusama/config.toml")

err = loadConfig(dotConfigToToml(cfg), kusamaConfigPath)
err = loadConfigFromFile(dotConfigToToml(cfg), kusamaConfigPath)
require.NoError(t, err)
}
2 changes: 1 addition & 1 deletion dot/core/service_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -288,7 +288,7 @@ func TestHandleChainReorg_WithReorg_Transactions(t *testing.T) {
t.Skip() // need to update this test to use a valid transaction

cfg := &Config{
Runtime: wasmer.NewTestInstance(t, runtime.NODE_RUNTIME),
Runtime: wasmer.NewTestInstance(t, runtime.WESTEND_RUNTIME_v0929),
}

s := NewTestService(t, cfg)
Expand Down
2 changes: 1 addition & 1 deletion dot/node_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,7 +458,7 @@ func TestNode_PersistGlobalName_WhenInitialize(t *testing.T) {
// newTestGenesisAndRuntime create a new test runtime and a new test genesis
// file with the test runtime stored in raw data and returns the genesis file
func newTestGenesisAndRuntime(t *testing.T) (filename string) {
runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.NODE_RUNTIME)
runtimeFilePath, err := runtime.GetRuntime(context.Background(), runtime.WESTEND_RUNTIME_v0929)
require.NoError(t, err)
runtimeData, err := os.ReadFile(runtimeFilePath)
require.NoError(t, err)
Expand Down
2 changes: 1 addition & 1 deletion dot/rpc/modules/dev_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ func newBABEService(t *testing.T) *babe.Service {

bs, es := newState(t)
tt := trie.NewEmptyTrie()
rt := wasmer.NewTestInstanceWithTrie(t, runtime.NODE_RUNTIME, tt)
rt := wasmer.NewTestInstanceWithTrie(t, runtime.WESTEND_RUNTIME_v0929, tt)
bs.StoreRuntime(bs.GenesisHash(), rt)
tt.Put(
common.MustHexToBytes("0x886726f904d8372fdabb7707870c2fad"),
Expand Down
43 changes: 43 additions & 0 deletions dot/rpc/modules/state.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ type StateStorageQueryRangeRequest struct {
EndBlock common.Hash `json:"block"`
}

// StateStorageQueryAtRequest holds json fields
type StateStorageQueryAtRequest struct {
Comment on lines +79 to +80
Copy link
Copy Markdown
Contributor

@qdm12 qdm12 Mar 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think StateStorageQueryAtRequest can be unexported

Copy link
Copy Markdown
Contributor

@kishansagathiya kishansagathiya Apr 2, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That would apply to its properties as well.

I don't see any variable of type StateStorageQueryAtRequest getting used by scale. Because scale requires structs and properties to be exported.

Copy link
Copy Markdown
Contributor

@qdm12 qdm12 Apr 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think they are exported for the encoding/json encoder to work right?

Copy link
Copy Markdown
Contributor Author

@edwardmack edwardmack Apr 3, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, they are exported for encoding/json encoder to work.

Copy link
Copy Markdown
Contributor

@qdm12 qdm12 Apr 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Still, is there a reason we keep StateStorageQueryAtRequest exported?

Copy link
Copy Markdown
Contributor Author

@edwardmack edwardmack Apr 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, we're using Gorilla web toolkit to expose functions through RPC, Gorilla exposes functions with a signature of containing exported structs for request and response. I tried changing StateStorageQueryAtRequest to un-exported and the RPC function was no longer accessible.

Keys []string `json:"keys" validate:"required"`
At common.Hash `json:"at"`
}

// StateStorageKeysQuery field to store storage keys
type StateStorageKeysQuery [][]byte

Expand Down Expand Up @@ -467,10 +473,13 @@ func (sm *StateModule) QueryStorage(
var hexValue *string
if len(value) > 0 {
hexValue = stringPtr(common.BytesToHex(value))
} else if value != nil { // empty byte slice value
hexValue = stringPtr("0x")
}

differentValueEncountered := i == startBlockNumber ||
lastValue[j] == nil && hexValue != nil ||
lastValue[j] != nil && hexValue == nil ||
lastValue[j] != nil && *lastValue[j] != *hexValue
if differentValueEncountered {
changes = append(changes, [2]*string{stringPtr(key), hexValue})
Expand All @@ -489,6 +498,40 @@ func (sm *StateModule) QueryStorage(
return nil
}

// QueryStorageAt queries historical storage entries (by key) at the block hash given or
// the best block if the given block hash is nil
func (sm *StateModule) QueryStorageAt(
_ *http.Request, request *StateStorageQueryAtRequest, response *[]StorageChangeSetResponse) error {
atBlockHash := request.At
if atBlockHash.IsEmpty() {
atBlockHash = sm.blockAPI.BestBlockHash()
}

changes := make([][2]*string, len(request.Keys))

for i, key := range request.Keys {
value, err := sm.storageAPI.GetStorageByBlockHash(&atBlockHash, common.MustHexToBytes(key))
if err != nil {
return fmt.Errorf("getting value by block hash: %w", err)
}
var hexValue *string
if len(value) > 0 {
hexValue = stringPtr(common.BytesToHex(value))
} else if value != nil { // empty byte slice value
hexValue = stringPtr("0x")
}

changes[i] = [2]*string{stringPtr(key), hexValue}
}

*response = []StorageChangeSetResponse{{
Copy link
Copy Markdown
Contributor

@qdm12 qdm12 Apr 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

TIL you don't need to initialize the slice pointer which is weird AF

Copy link
Copy Markdown
Contributor Author

@edwardmack edwardmack Apr 5, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this is because response is initialized since it is a function argument.

Block: &atBlockHash,
Changes: changes,
}}

return nil
}

func stringPtr(s string) *string { return &s }

// SubscribeRuntimeVersion initialised a runtime version subscription and returns the current version
Expand Down
32 changes: 19 additions & 13 deletions dot/rpc/modules/state_integration_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,29 +28,35 @@ const (
)

func TestStateModule_GetRuntimeVersion(t *testing.T) {
// expected results based on responses from prior tests
/* expected results based on responses from prior tests
We can get this data from polkadot runtime release source code
https://github.com/paritytech/polkadot/blob/v0.9.29/runtime/westend/src/lib.rs#L105-L117
*/
expected := StateRuntimeVersionResponse{
SpecName: "node",
ImplName: "substrate-node",
AuthoringVersion: 10,
SpecVersion: 264,
SpecName: "westend",
ImplName: "parity-westend",
AuthoringVersion: 2,
SpecVersion: 9290,
ImplVersion: 0,
Apis: []interface{}{
[]interface{}{"0xdf6acb689907609b", uint32(3)},
[]interface{}{"0xdf6acb689907609b", uint32(4)},
[]interface{}{"0x37e397fc7c91f5e4", uint32(1)},
[]interface{}{"0x40fe3ad401f8959a", uint32(4)},
[]interface{}{"0xd2bc9897eed08f15", uint32(2)},
[]interface{}{"0x40fe3ad401f8959a", uint32(6)},
[]interface{}{"0xd2bc9897eed08f15", uint32(3)},
[]interface{}{"0xf78b278be53f454c", uint32(2)},
[]interface{}{"0xed99c5acb25eedf5", uint32(2)},
[]interface{}{"0xaf2c0297a23e6d3d", uint32(2)},
[]interface{}{"0x49eaaf1b548a0cb0", uint32(1)},
[]interface{}{"0x91d5df18b0d2cf58", uint32(1)},
[]interface{}{"0xed99c5acb25eedf5", uint32(3)},
[]interface{}{"0xcbca25e39f142387", uint32(2)},
[]interface{}{"0x687ad44ad37f03c2", uint32(1)},
[]interface{}{"0xab3c0572291feb8b", uint32(1)},
[]interface{}{"0xbc9d89904f5b923f", uint32(1)},
[]interface{}{"0x68b66ba122c93fa7", uint32(1)},
[]interface{}{"0x37c8bb1350a9a2a8", uint32(1)},
[]interface{}{"0x91d5df18b0d2cf58", uint32(1)},
[]interface{}{"0xab3c0572291feb8b", uint32(1)},
[]interface{}{"0xf3ff14d5ab527059", uint32(1)},
[]interface{}{"0x17a6bc0d0062aeb3", uint32(1)},
},
TransactionVersion: 2,
TransactionVersion: 12,
}

sm, hash, _ := setupStateModule(t)
Expand Down
Loading