Add --experimental_src_map to ethier gen command (#35)

* Solidity source mapping from OpCodes at runtime!

* Fix test failure introduced by upgrade of go-ethereum + bring new test code in line with implementation variable names.

* Run prettier on Solidity test contract

* Properly handle source-code resolution from EVM trace when executing libraries.

* GitHub test Action is failing despite tests passing so attempting to change the node_modules cache key to clear it.

* Revert GitHub Action cache key and change testing to non-verbose as it hides failures; also output `solc` + `abigen` versions.

* Deliberate panic in ethier/gen.go to diagnose difference from local run to GitHub Actions

* A further deliberate panic in ethier/gen.go to see full output; abigen differs and regexp is a bad idea here because it's too fragile

* Update ethier/gen.go to match output of old and new versions of abigen after ethereum/go-ethereum#24835.

See the TODO in ethier/gen.go re direct modification of the bind.Bind() template.

* Simplify calculation of PUSH<N> instruction offset

Author: aschlosberg

.github/workflows/test.yml

@@ -40,4 +40,4 @@ jobs:
           sudo apt-get install -y solc abigen
 
       - name: Run tests
-        run: npm run test:verbose
+        run: npm run test

eth/signer_test.go

@@ -14,6 +14,7 @@ import (
 	"github.com/google/tink/go/keyset"
 	"github.com/google/tink/go/prf"
 	"github.com/google/tink/go/tink"
+	"github.com/h-fam/errdiff"
 
 	// These tests require ethtest.SimulatedBackend but that would result in a
 	// cyclical dependency. As this is limited to these tests and not the
@@ -161,19 +162,19 @@ func TestTransactorWithChainID(t *testing.T) {
 	const gasLimit = 21000
 	txFee := new(big.Int).Mul(gasPrice, big.NewInt(gasLimit))
 
-	sendEth := func(t *testing.T, opts *bind.TransactOpts, to common.Address, value *big.Int) {
+	sendEth := func(t *testing.T, opts *bind.TransactOpts, to common.Address, value *big.Int, errDiffAgainst interface{}) {
 		t.Helper()
 		unsigned := types.NewTransaction(0, to, value, gasLimit, gasPrice, nil)
 		tx, err := opts.Signer(opts.From, unsigned)
 		if err != nil {
 			t.Fatalf("%T.Signer(%+v) error %v", opts, unsigned, err)
 		}
-		if err := sim.SendTransaction(ctx, tx); err != nil {
-			t.Fatalf("%T.SendTransaction() error %v", sim, err)
+		if diff := errdiff.Check(sim.SendTransaction(ctx, tx), errDiffAgainst); diff != "" {
+			t.Fatalf("%T.SendTransaction() %s", sim, diff)
 		}
 	}
 
-	sendEth(t, sim.Acc(0), signer.Address(), Ether(42))
+	sendEth(t, sim.Acc(0), signer.Address(), Ether(42), nil)
 	wantBalance(ctx, t, "faucet after sending 42", sim.Addr(0), new(big.Int).Sub(Ether(100-42), txFee))
 	wantBalance(ctx, t, "signer after receiving 42", signer.Address(), Ether(42))
 
@@ -183,32 +184,17 @@ func TestTransactorWithChainID(t *testing.T) {
 		if err != nil {
 			t.Fatalf("%T.TransactorWithChainID(%d) error %v", signer, chainID, err)
 		}
-		sendEth(t, opts, sim.Addr(0), Ether(21))
+		sendEth(t, opts, sim.Addr(0), Ether(21), nil)
 		wantBalance(ctx, t, "faucet after sending 42 and receiving 21", sim.Addr(0), new(big.Int).Sub(Ether(100-42+21), txFee))
 		wantBalance(ctx, t, "signer after receiving 42 and sending 21", signer.Address(), new(big.Int).Sub(Ether(42-21), txFee))
 	})
 
 	t.Run("incorrect chain ID", func(t *testing.T) {
-		// The SimulatedBackend panics instead of returning an error when the
-		// chain ID is incorrect. #java
-		defer func() {
-			const wantContains = "invalid chain id"
-			r := recover()
-
-			if err, ok := r.(error); ok && strings.Contains(err.Error(), wantContains) {
-				return
-			}
-			t.Errorf("%T.SendTransaction(%T.TransactorWithChainID(<incorrect ID>)) recovered %T(%v); want panic with error containing %q", sim, signer, r, r, wantContains)
-		}()
-
 		chainID := new(big.Int).Add(sim.Blockchain().Config().ChainID, big.NewInt(1))
 		opts, err := signer.TransactorWithChainID(chainID)
 		if err != nil {
 			t.Fatalf("%T.TransactorWithChainID(%d) error %v", signer, chainID, err)
 		}
-		sendEth(t, opts, sim.Addr(0), Ether(1))
-		// We should never reach here because sendEth results in a panic inside
-		// go-ethereum's SimulatedBackend.
-		t.Errorf("%T.SendTransaction(%T.TransactorWithChainID(<incorrect ID>)) did not panic", sim, signer)
+		sendEth(t, opts, sim.Addr(0), Ether(1), "invalid chain id")
 	})
 }

ethier/gen.go

@@ -1,20 +1,33 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"go/format"
+	"go/parser"
+	"go/token"
 	"io"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
+	"text/template"
 
+	"github.com/ethereum/go-ethereum/common/compiler"
 	"github.com/spf13/cobra"
+	"golang.org/x/tools/go/ast/astutil"
+
+	_ "embed"
 )
 
+const srcMapFlag = "experimental_src_map"
+
 func init() {
-	rootCmd.AddCommand(&cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "gen",
 		Short: "Compiles Solidity contracts to generate Go ABI bindings with go:generate",
 		RunE:  gen,
@@ -29,12 +42,16 @@ func init() {
 			}
 			return nil
 		},
-	})
+	}
+
+	cmd.Flags().Bool(srcMapFlag, false, "Generate source maps to determine Solidity code location from EVM traces")
+
+	rootCmd.AddCommand(cmd)
 }
 
 // gen runs `solc | abigen` on the Solidity source files passed as the args.
 // TODO: support wildcard / glob matching of files.
-func gen(_ *cobra.Command, args []string) (retErr error) {
+func gen(cmd *cobra.Command, args []string) (retErr error) {
 	pwd, err := os.Getwd()
 	if err != nil {
 		return fmt.Errorf("os.Getwd(): %v", err)
@@ -62,7 +79,7 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
 		args,
 		"--base-path", basePath,
 		"--include-path", filepath.Join(basePath, "node_modules"),
-		"--combined-json", "abi,bin",
+		"--combined-json", "abi,bin,bin-runtime,hashes,srcmap-runtime",
 	)
 	solc := exec.Command("solc", args...)
 	solc.Stderr = os.Stderr
@@ -75,13 +92,16 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
 		"abigen",
 		"--combined-json", "/dev/stdin",
 		"--pkg", pkg,
-		"--out", "generated.go",
 	)
 	abigen.Stderr = os.Stderr
 
 	r, w := io.Pipe()
 	solc.Stdout = w
-	abigen.Stdin = r
+	combinedJSON := bytes.NewBuffer(nil)
+	abigen.Stdin = io.TeeReader(r, combinedJSON)
+
+	generated := bytes.NewBuffer(nil)
+	abigen.Stdout = generated
 
 	if err := solc.Start(); err != nil {
 		return fmt.Errorf("start `solc`: %v", err)
@@ -99,5 +119,146 @@ func gen(_ *cobra.Command, args []string) (retErr error) {
 	if err := abigen.Wait(); err != nil {
 		return fmt.Errorf("`abigen` returned: %v", err)
 	}
-	return r.Close()
+	if err := r.Close(); err != nil {
+		return fmt.Errorf("close read-half of pipe from solc to abigen: %v", err)
+	}
+
+	extend, err := cmd.Flags().GetBool(srcMapFlag)
+	if err != nil {
+		return fmt.Errorf("%T.Flags().GetBool(%q): %v", cmd, srcMapFlag, err)
+	}
+	if !extend {
+		return os.WriteFile("generated.go", generated.Bytes(), 0644)
+	}
+
+	out, err := extendGeneratedCode(generated, combinedJSON)
+	if err != nil {
+		return err
+	}
+	return os.WriteFile("generated.go", out, 0644)
+}
+
+var (
+	//go:embed gen_extra.go.tmpl
+	extraCode string
+
+	// extraTemplate is the template for use by extendGeneratedCode().
+	extraTemplate = template.Must(
+		template.New("extra").
+			Funcs(template.FuncMap{
+				"quote": func(s interface{}) string {
+					return fmt.Sprintf("%q", s)
+				},
+				"stringSlice": func(strs []string) string {
+					q := make([]string, len(strs))
+					for i, s := range strs {
+						q[i] = fmt.Sprintf("%q", s)
+					}
+					return fmt.Sprintf("[]string{%s}", strings.Join(q, ", "))
+				},
+				"contract": func(s string) (string, error) {
+					parts := strings.Split(s, ".sol:")
+					if len(parts) != 2 {
+						return "", fmt.Errorf("invalid contract name %q must have format path/to/file.sol:ContractName", s)
+					}
+					return parts[1], nil
+				},
+			}).
+			Parse(extraCode),
+	)
+
+	// Regular expressions for modifying abigen-generated code to work with the
+	// extraTemplate code above.
+	deployedRegexp = regexp.MustCompile(`^\s*return address, tx, &(.+?)\{.*Transactor.*\}, nil\s*$`)
+	// Note the option for matching strings.Replace or strings.ReplaceAll due to
+	// a recent change in abigen.
+	libReplacementRegexp = regexp.MustCompile(`^\s*(.+?)Bin = strings.Replace(?:All)?\(.+?, "__\$([0-9a-f]{34})\$__", (.+?)(?:, -1)?\)\s*$`)
+	// TODO(aschlosberg) replace regular expressions with a more explicit
+	// approach for modifying the output code. This likely requires a PR to the
+	// go-ethereum repo to allow bind.Bind (+/- abigen) to accept an alternate
+	// template.
+)
+
+// extendGeneratedCode adds ethier-specific functionality to code generated by
+// abigen, allowing for interoperability with the ethier/solidity package for
+// source-map interpretation at runtime.
+func extendGeneratedCode(generated, combinedJSON *bytes.Buffer) ([]byte, error) {
+	meta := struct {
+		SourceList []string `json:"sourceList"`
+		Version    string   `json:"version"`
+
+		Contracts    map[string]*compiler.Contract
+		CombinedJSON string
+	}{CombinedJSON: combinedJSON.String()}
+
+	if err := json.Unmarshal(combinedJSON.Bytes(), &meta); err != nil {
+		return nil, fmt.Errorf("json.Unmarshal([solc output], %T): %v", &meta, err)
+	}
+
+	cs, err := compiler.ParseCombinedJSON(combinedJSON.Bytes(), "", "", meta.Version, "")
+	if err != nil {
+		return nil, fmt.Errorf("compiler.ParseCombinedJSON(): %v", err)
+	}
+	meta.Contracts = cs
+	for k, c := range meta.Contracts {
+		if c.RuntimeCode == "0x" {
+			delete(meta.Contracts, k)
+		}
+	}
+
+	if err := extraTemplate.Execute(generated, meta); err != nil {
+		return nil, fmt.Errorf("%T.Execute(): %v", extraTemplate, err)
+	}
+
+	// When using vm.Config.Trace, the only contract-identifying information is
+	// the address to which the transaction was sent. We must therefore modify
+	// every DeployFoo() function to save the address(es) at which the contract
+	// is deployed.
+	lines := strings.Split(generated.String(), "\n")
+	for i, l := range lines {
+		matches := deployedRegexp.FindStringSubmatch(l)
+		if len(matches) == 0 {
+			continue
+		}
+		lines[i] = fmt.Sprintf(
+			`deployedContracts[address] = %q // Added by ethier gen
+			%s`,
+			matches[1], l,
+		)
+	}
+
+	// Libraries have their addresses string-replaced directly into contract
+	// code, which we need to mirror for the runtime code too.
+	for i, l := range lines {
+		matches := libReplacementRegexp.FindStringSubmatch(l)
+		if len(matches) == 0 {
+			continue
+		}
+		lines[i] = fmt.Sprintf(
+			`%s
+			RuntimeSourceMaps[%q].RuntimeCode = strings.Replace(RuntimeSourceMaps[%[2]q].RuntimeCode, "__$%s$__", %s, -1)`,
+			l, matches[1], matches[2], matches[3],
+		)
+	}
+
+	// Effectively the same as running goimports on the (ugly) generated code.
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "generated.go", strings.Join(lines, "\n"), parser.ParseComments|parser.AllErrors)
+	if err != nil {
+		return nil, fmt.Errorf("parser.ParseFile(%T, …): %v", fset, err)
+	}
+	for _, pkg := range []string{
+		"github.com/ethereum/go-ethereum/common/compiler",
+		"github.com/divergencetech/ethier/solidity",
+	} {
+		if !astutil.AddImport(fset, f, pkg) {
+			return nil, fmt.Errorf("add import %q to generated Go: %v", pkg, err)
+		}
+	}
+
+	buf := bytes.NewBuffer(nil)
+	if err := format.Node(buf, fset, f); err != nil {
+		return nil, fmt.Errorf("format.Node(%T, %T, %T): %v", buf, fset, f, err)
+	}
+	return buf.Bytes(), nil
 }

ethier/gen_extra.go.tmpl

@@ -0,0 +1,52 @@
+/**
+ *
+ * Additional code added by ethier, beyond standard abigen output.
+ *
+ */
+
+const (
+    // SolCVersion is the version of the Solidity compiler used to create this
+    // file.
+    SolCVersion = {{quote .Version}}
+    
+    // CombinedJSON is the raw combined-JSON output of solc, passed to abigen to
+    // create this file.
+    CombinedJSON = {{quote .CombinedJSON}}
+)
+
+var (
+    // SourceList is the list of source files used by solc when compiling these
+    // contracts. Their indices correspond to the file numbers in the source
+    // maps.
+    SourceList = {{stringSlice .SourceList}}
+
+    // RuntimeSourceMaps contains, for each compiled contract, the runtime
+    // binary and its associated source map. With a program counter pointing to
+    // an instruction in the runtime binary, this is sufficient to determine the
+    // respective location in the Solidity code from which the binary was
+    // compiled.
+    RuntimeSourceMaps = map[string]*compiler.Contract{
+    {{- range $src, $c := .Contracts }}
+    {{quote (contract $src)}}: {
+        RuntimeCode: {{quote $c.RuntimeCode}},
+        Info: compiler.ContractInfo{
+            SrcMapRuntime: {{quote $c.Info.SrcMapRuntime}},
+        },
+    },
+    {{- end }}
+    }
+)
+
+// deployedContracts tracks which contract is deployed at each address. The
+// standard abigen Deploy<ContractName>() functions have been modified to set
+// the value of this map to <ContractName> before returning the deployment
+// address. This allows SourceMap() to function correctly.
+var deployedContracts = make(map[common.Address]string)
+
+// SourceMap returns a new SourceMap, able to convert program counters to
+// Solidity source offsets. SourceMap() must be called after contracts are
+// deployed otherwise they won't be registered by contract address (only by
+// contract name).
+func SourceMap() (*solidity.SourceMap, error) {
+    return solidity.NewSourceMap(SourceList, RuntimeSourceMaps, deployedContracts)
+}