From 52e8f2ad55186b533348c7178e5a6b4e27e86444 Mon Sep 17 00:00:00 2001 From: Dan Forbes Date: Wed, 19 Jan 2022 14:15:52 -0800 Subject: [PATCH 1/4] Support Substrate Wasm compression https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs --- lib/runtime/wasmer/instance.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index 7c87ceb287..d996029d96 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -4,6 +4,7 @@ package wasmer import ( + "bytes" "errors" "fmt" "sync" @@ -18,6 +19,8 @@ import ( "github.com/ChainSafe/gossamer/lib/crypto" wasm "github.com/wasmerio/go-ext-wasm/wasmer" + + "github.com/klauspost/compress/zstd" ) // Name represents the name of the interpreter @@ -94,6 +97,22 @@ func NewInstance(code []byte, cfg *Config) (*Instance, error) { return nil, errors.New("code is empty") } + // Substrate Wasm compression + // https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs + compressionFlag := []byte{82, 188, 83, 118, 70, 219, 142, 5} + if bytes.HasPrefix(code, compressionFlag) { + logger.Debug("Decompressing Wasm runtime code") + var decoder, err = zstd.NewReader(nil) + if err != nil { + return nil, errors.New("failed to create zstd decoder") + } + + code, err = decoder.DecodeAll(code[len(compressionFlag):], nil) + if err != nil { + return nil, errors.New("failed to decompress Wasm") + } + } + logger.Patch(log.SetLevel(cfg.LogLvl), log.SetCallerFunc(true)) imports, err := cfg.Imports() From 3abdd2bde0f0fb8c6b15acd9a0219f14d5aa0403 Mon Sep 17 00:00:00 2001 From: Dan Forbes Date: Thu, 20 Jan 2022 05:57:43 -0800 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: noot <36753753+noot@users.noreply.github.com> --- lib/runtime/wasmer/instance.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index d996029d96..5028d786b1 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -99,17 +99,17 @@ func NewInstance(code []byte, cfg *Config) (*Instance, error) { // Substrate Wasm compression // https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs - compressionFlag := []byte{82, 188, 83, 118, 70, 219, 142, 5} + const compressionFlag = []byte{82, 188, 83, 118, 70, 219, 142, 5} if bytes.HasPrefix(code, compressionFlag) { logger.Debug("Decompressing Wasm runtime code") - var decoder, err = zstd.NewReader(nil) + decoder, err := zstd.NewReader(nil) if err != nil { - return nil, errors.New("failed to create zstd decoder") + return nil, fmt.Errorf("failed to create zstd decoder: %w", err) } code, err = decoder.DecodeAll(code[len(compressionFlag):], nil) if err != nil { - return nil, errors.New("failed to decompress Wasm") + return nil, fmt.Errorf("failed to decompress compressed wasm code: %w", err) } } From 7482560ec3d40ba338accf0e0ab2660925925b77 Mon Sep 17 00:00:00 2001 From: Dan Forbes Date: Thu, 20 Jan 2022 16:48:22 -0800 Subject: [PATCH 3/4] Review comments - slices cannot be const - create function & write tests - `go mod tidy` --- go.mod | 2 +- lib/runtime/wasmer/instance.go | 34 +++++++++++++++----------- lib/runtime/wasmer/instance_test.go | 38 +++++++++++++++++++++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 120ef2e983..4f192034d3 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,7 @@ require ( github.com/ipfs/go-ds-badger2 v0.1.1 github.com/ipfs/go-ipns v0.1.2 //indirect github.com/jpillora/ipfilter v1.2.3 + github.com/klauspost/compress v1.12.3 github.com/libp2p/go-libp2p v0.15.1 github.com/libp2p/go-libp2p-core v0.9.0 github.com/libp2p/go-libp2p-discovery v0.5.1 @@ -84,7 +85,6 @@ require ( github.com/jbenet/goprocess v0.1.4 // indirect github.com/jcelliott/lumber v0.0.0-20160324203708-dd349441af25 // indirect github.com/jpillora/backoff v1.0.0 // indirect - github.com/klauspost/compress v1.12.3 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/koron/go-ssdp v0.0.2 // indirect github.com/leodido/go-urn v1.2.1 // indirect diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index 5028d786b1..5fc73b64a2 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -97,20 +97,10 @@ func NewInstance(code []byte, cfg *Config) (*Instance, error) { return nil, errors.New("code is empty") } - // Substrate Wasm compression - // https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs - const compressionFlag = []byte{82, 188, 83, 118, 70, 219, 142, 5} - if bytes.HasPrefix(code, compressionFlag) { - logger.Debug("Decompressing Wasm runtime code") - decoder, err := zstd.NewReader(nil) - if err != nil { - return nil, fmt.Errorf("failed to create zstd decoder: %w", err) - } - - code, err = decoder.DecodeAll(code[len(compressionFlag):], nil) - if err != nil { - return nil, fmt.Errorf("failed to decompress compressed wasm code: %w", err) - } + var err error + code, err = decompressWasm(code) + if err != nil { + return nil, err } logger.Patch(log.SetLevel(cfg.LogLvl), log.SetCallerFunc(true)) @@ -176,6 +166,22 @@ func NewInstance(code []byte, cfg *Config) (*Instance, error) { return inst, nil } +// decompressWasm decompresses a Wasm blob that may or may not be compressed with zstd +// ref: https://github.com/paritytech/substrate/blob/master/primitives/maybe-compressed-blob/src/lib.rs +func decompressWasm(code []byte) ([]byte, error) { + compressionFlag := []byte{82, 188, 83, 118, 70, 219, 142, 5} + if !bytes.HasPrefix(code, compressionFlag) { + return code, nil + } + + decoder, err := zstd.NewReader(nil) + if err != nil { + return nil, fmt.Errorf("failed to create zstd decoder: %w", err) + } + + return decoder.DecodeAll(code[len(compressionFlag):], nil) +} + // GetCodeHash returns the code of the instance func (in *Instance) GetCodeHash() common.Hash { return in.codeHash diff --git a/lib/runtime/wasmer/instance_test.go b/lib/runtime/wasmer/instance_test.go index 08e071223e..81413a610c 100644 --- a/lib/runtime/wasmer/instance_test.go +++ b/lib/runtime/wasmer/instance_test.go @@ -10,6 +10,8 @@ import ( "github.com/ChainSafe/gossamer/lib/runtime" "github.com/stretchr/testify/require" + + "github.com/klauspost/compress/zstd" ) // test used for ensuring runtime exec calls can me made concurrently @@ -63,3 +65,39 @@ func TestInstance_CheckRuntimeVersion(t *testing.T) { require.Equal(t, expected.ImplVersion(), version.ImplVersion()) require.Equal(t, expected.TransactionVersion(), version.TransactionVersion()) } + +func TestDecompressWasm(t *testing.T) { + encoder, _ := zstd.NewWriter(nil) + cases := []struct { + in []byte + expected []byte + msg string + }{ + { + []byte{82, 188, 83, 118, 70, 219, 142}, + []byte{82, 188, 83, 118, 70, 219, 142}, + "partial compression flag", + }, + { + []byte{82, 188, 83, 118, 70, 219, 142, 6}, + []byte{82, 188, 83, 118, 70, 219, 142, 6}, + "wrong compression flag", + }, + { + []byte{82, 188, 83, 118, 70, 219, 142, 6, 221}, + []byte{82, 188, 83, 118, 70, 219, 142, 6, 221}, + "wrong compression flag with data", + }, + { + append([]byte{82, 188, 83, 118, 70, 219, 142, 5}, encoder.EncodeAll([]byte("compressed"), make([]byte, 0))...), + []byte("compressed"), + "compressed data", + }, + } + + for _, test := range cases { + actual, err := decompressWasm(test.in) + require.NoError(t, err) + require.Equal(t, test.expected, actual) + } +} From 53b35af48d6349eeb0a8edf269458538236f74a8 Mon Sep 17 00:00:00 2001 From: Dan Forbes Date: Fri, 21 Jan 2022 07:15:10 -0800 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Quentin McGaw --- lib/runtime/wasmer/instance.go | 2 +- lib/runtime/wasmer/instance_test.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/runtime/wasmer/instance.go b/lib/runtime/wasmer/instance.go index 5fc73b64a2..da3b5441a9 100644 --- a/lib/runtime/wasmer/instance.go +++ b/lib/runtime/wasmer/instance.go @@ -100,7 +100,7 @@ func NewInstance(code []byte, cfg *Config) (*Instance, error) { var err error code, err = decompressWasm(code) if err != nil { - return nil, err + return nil, fmt.Errorf("cannot decompress WASM code: %w", err) } logger.Patch(log.SetLevel(cfg.LogLvl), log.SetCallerFunc(true)) diff --git a/lib/runtime/wasmer/instance_test.go b/lib/runtime/wasmer/instance_test.go index 81413a610c..89b95d70eb 100644 --- a/lib/runtime/wasmer/instance_test.go +++ b/lib/runtime/wasmer/instance_test.go @@ -67,7 +67,9 @@ func TestInstance_CheckRuntimeVersion(t *testing.T) { } func TestDecompressWasm(t *testing.T) { - encoder, _ := zstd.NewWriter(nil) + encoder, err := zstd.NewWriter(nil) + require.NoError(t, err) + cases := []struct { in []byte expected []byte @@ -89,7 +91,7 @@ func TestDecompressWasm(t *testing.T) { "wrong compression flag with data", }, { - append([]byte{82, 188, 83, 118, 70, 219, 142, 5}, encoder.EncodeAll([]byte("compressed"), make([]byte, 0))...), + append([]byte{82, 188, 83, 118, 70, 219, 142, 5}, encoder.EncodeAll([]byte("compressed"), nil)...), []byte("compressed"), "compressed data", },