rlp/rlpgen: RLP encoder code generator

This is a new tool to generate EncodeRLP (and DecodeRLP) method
implementations from a struct definition.

Author: fjl

rlp/rlpgen/gen.go

@@ -0,0 +1,92 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"go/ast"
+	"go/importer"
+	"go/parser"
+	"go/token"
+	"go/types"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// Package RLP is loaded only once and reused for all tests.
+var (
+	testFset       = token.NewFileSet()
+	testImporter   = importer.ForCompiler(testFset, "source", nil).(types.ImporterFrom)
+	testPackageRLP *types.Package
+)
+
+func init() {
+	cwd, err := os.Getwd()
+	if err != nil {
+		panic(err)
+	}
+	testPackageRLP, err = testImporter.ImportFrom(pathOfPackageRLP, cwd, 0)
+	if err != nil {
+		panic(fmt.Errorf("can't load package RLP: %v", err))
+	}
+}
+
+var tests = []string{"uints", "nil", "rawvalue", "optional", "bigint"}
+
+func TestOutput(t *testing.T) {
+	for _, test := range tests {
+		test := test
+		t.Run(test, func(t *testing.T) {
+			inputFile := filepath.Join("testdata", test+".in.txt")
+			outputFile := filepath.Join("testdata", test+".out.txt")
+			bctx, typ, err := loadTestSource(inputFile, "Test")
+			if err != nil {
+				t.Fatal("error loading test source:", err)
+			}
+			output, err := bctx.generate(typ, true, true)
+			if err != nil {
+				t.Fatal("error in generate:", err)
+			}
+
+			// Set this environment variable to regenerate the test outputs.
+			if os.Getenv("WRITE_TEST_FILES") != "" {
+				ioutil.WriteFile(outputFile, output, 0644)
+			}
+
+			// Check if output matches.
+			wantOutput, err := ioutil.ReadFile(outputFile)
+			if err != nil {
+				t.Fatal("error loading expected test output:", err)
+			}
+			if !bytes.Equal(output, wantOutput) {
+				t.Fatal("output mismatch:\n", string(output))
+			}
+		})
+	}
+}
+
+func loadTestSource(file string, typeName string) (*buildContext, *types.Named, error) {
+	// Load the test input.
+	content, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, nil, err
+	}
+	f, err := parser.ParseFile(testFset, file, content, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	conf := types.Config{Importer: testImporter}
+	pkg, err := conf.Check("test", testFset, []*ast.File{f}, nil)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	// Find the test struct.
+	bctx := newBuildContext(testPackageRLP)
+	typ, err := lookupStructType(pkg.Scope(), typeName)
+	if err != nil {
+		return nil, nil, fmt.Errorf("can't find type %s: %v", typeName, err)
+	}
+	return bctx, typ, nil
+}

rlp/rlpgen/gen_test.go

@@ -0,0 +1,148 @@
+// Copyright 2021 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"bytes"
+	"errors"
+	"flag"
+	"fmt"
+	"go/types"
+	"io/ioutil"
+	"os"
+
+	"golang.org/x/tools/go/packages"
+)
+
+const pathOfPackageRLP = "github.com/ethereum/go-ethereum/rlp"
+
+func main() {
+	var (
+		pkgdir     = flag.String("dir", ".", "input package")
+		output     = flag.String("out", "-", "output file (default is stdout)")
+		genEncoder = flag.Bool("encoder", true, "generate EncodeRLP?")
+		genDecoder = flag.Bool("decoder", false, "generate DecodeRLP?")
+		typename   = flag.String("type", "", "type to generate methods for")
+	)
+	flag.Parse()
+
+	cfg := Config{
+		Dir:             *pkgdir,
+		Type:            *typename,
+		GenerateEncoder: *genEncoder,
+		GenerateDecoder: *genDecoder,
+	}
+	code, err := cfg.process()
+	if err != nil {
+		fatal(err)
+	}
+	if *output == "-" {
+		os.Stdout.Write(code)
+	} else if err := ioutil.WriteFile(*output, code, 0644); err != nil {
+		fatal(err)
+	}
+}
+
+func fatal(args ...interface{}) {
+	fmt.Fprintln(os.Stderr, args...)
+	os.Exit(1)
+}
+
+type Config struct {
+	Dir  string // input package directory
+	Type string
+
+	GenerateEncoder bool
+	GenerateDecoder bool
+}
+
+// process generates the Go code.
+func (cfg *Config) process() (code []byte, err error) {
+	// Load packages.
+	pcfg := &packages.Config{
+		Mode:       packages.NeedName | packages.NeedTypes | packages.NeedImports | packages.NeedDeps,
+		Dir:        cfg.Dir,
+		BuildFlags: []string{"-tags", "norlpgen"},
+	}
+	ps, err := packages.Load(pcfg, pathOfPackageRLP, ".")
+	if err != nil {
+		return nil, err
+	}
+	if len(ps) == 0 {
+		return nil, fmt.Errorf("no Go package found in %s", cfg.Dir)
+	}
+	packages.PrintErrors(ps)
+
+	// Find the packages that were loaded.
+	var (
+		pkg        *types.Package
+		packageRLP *types.Package
+	)
+	for _, p := range ps {
+		if len(p.Errors) > 0 {
+			return nil, fmt.Errorf("package %s has errors", p.PkgPath)
+		}
+		if p.PkgPath == pathOfPackageRLP {
+			packageRLP = p.Types
+		} else {
+			pkg = p.Types
+		}
+	}
+	bctx := newBuildContext(packageRLP)
+
+	// Find the type and generate.
+	typ, err := lookupStructType(pkg.Scope(), cfg.Type)
+	if err != nil {
+		return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err)
+	}
+	code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder)
+	if err != nil {
+		return nil, err
+	}
+
+	// Add build comments.
+	// This is done here to avoid processing these lines with gofmt.
+	var header bytes.Buffer
+	fmt.Fprint(&header, "// Code generated by rlpgen. DO NOT EDIT.\n\n")
+	fmt.Fprint(&header, "//go:build !norlpgen\n")
+	fmt.Fprint(&header, "// +build !norlpgen\n\n")
+	return append(header.Bytes(), code...), nil
+}
+
+func lookupStructType(scope *types.Scope, name string) (*types.Named, error) {
+	typ, err := lookupType(scope, name)
+	if err != nil {
+		return nil, err
+	}
+	_, ok := typ.Underlying().(*types.Struct)
+	if !ok {
+		return nil, errors.New("not a struct type")
+	}
+	return typ, nil
+}
+
+func lookupType(scope *types.Scope, name string) (*types.Named, error) {
+	obj := scope.Lookup(name)
+	if obj == nil {
+		return nil, errors.New("no such identifier")
+	}
+	typ, ok := obj.(*types.TypeName)
+	if !ok {
+		return nil, errors.New("not a type")
+	}
+	return typ.Type().(*types.Named), nil
+}

rlp/rlpgen/main.go

@@ -0,0 +1,10 @@
+// -*- mode: go -*-
+
+package test
+
+import "math/big"
+
+type Test struct {
+	Int      *big.Int
+	IntNoPtr big.Int
+}

rlp/rlpgen/testdata/bigint.in.txt

@@ -0,0 +1,49 @@
+package test
+
+import "github.com/ethereum/go-ethereum/rlp"
+import "io"
+
+func (obj *Test) EncodeRLP(_w io.Writer) error {
+	w := rlp.NewEncoderBuffer(_w)
+	_tmp0 := w.List()
+	if obj.Int == nil {
+		w.Write(rlp.EmptyString)
+	} else {
+		if obj.Int.Sign() == -1 {
+			return rlp.ErrNegativeBigInt
+		}
+		w.WriteBigInt(obj.Int)
+	}
+	if obj.IntNoPtr.Sign() == -1 {
+		return rlp.ErrNegativeBigInt
+	}
+	w.WriteBigInt(&obj.IntNoPtr)
+	w.ListEnd(_tmp0)
+	return w.Flush()
+}
+
+func (obj *Test) DecodeRLP(dec *rlp.Stream) error {
+	var _tmp0 Test
+	{
+		if _, err := dec.List(); err != nil {
+			return err
+		}
+		// Int:
+		_tmp1, err := dec.BigInt()
+		if err != nil {
+			return err
+		}
+		_tmp0.Int = _tmp1
+		// IntNoPtr:
+		_tmp2, err := dec.BigInt()
+		if err != nil {
+			return err
+		}
+		_tmp0.IntNoPtr = (*_tmp2)
+		if err := dec.ListEnd(); err != nil {
+			return err
+		}
+	}
+	*obj = _tmp0
+	return nil
+}

rlp/rlpgen/testdata/bigint.out.txt

@@ -0,0 +1,30 @@
+// -*- mode: go -*-
+
+package test
+
+type Aux struct{
+    A uint32
+}
+
+type Test struct{
+    Uint8 *byte `rlp:"nil"`
+    Uint8List *byte `rlp:"nilList"`
+
+    Uint32 *uint32 `rlp:"nil"`
+    Uint32List *uint32 `rlp:"nilList"`
+
+    Uint64 *uint64 `rlp:"nil"`
+    Uint64List *uint64 `rlp:"nilList"`
+
+    String *string `rlp:"nil"`
+    StringList *string `rlp:"nilList"`
+
+    ByteArray *[3]byte `rlp:"nil"`
+    ByteArrayList *[3]byte `rlp:"nilList"`
+
+    ByteSlice *[]byte `rlp:"nil"`
+    ByteSliceList *[]byte `rlp:"nilList"`
+
+    Struct *Aux `rlp:"nil"`
+    StructString *Aux `rlp:"nilString"`
+}