WIP - writing state diff to a CSV

Author: elizabethengelman

statediff/config.go

@@ -30,7 +30,8 @@ type Config struct {
 type StateDiffMode int
 
 const (
-	IPLD  StateDiffMode = iota
+	CSV  StateDiffMode = iota
+	IPLD
 	LDB
 	SQL
 )
@@ -42,6 +43,8 @@ func (mode StateDiffMode) IsValid() bool {
 // String implements the stringer interface.
 func (mode StateDiffMode) String() string {
 	switch mode {
+	case CSV:
+		return "csv"
 	case IPLD:
 		return "ipfs"
 	case LDB:
@@ -55,6 +58,8 @@ func (mode StateDiffMode) String() string {
 
 func (mode StateDiffMode) MarshalText() ([]byte, error) {
 	switch mode {
+	case CSV:
+		return []byte("ipfs"), nil
 	case IPLD:
 		return []byte("ipfs"), nil
 	case LDB:
@@ -68,6 +73,8 @@ func (mode StateDiffMode) MarshalText() ([]byte, error) {
 
 func (mode *StateDiffMode) UnmarshalText(text []byte) error {
 	switch string(text) {
+	case "csv":
+		*mode = CSV
 	case "ipfs":
 		*mode = IPLD
 	case "ldb":

statediff/publisher.go

@@ -20,44 +20,166 @@
 package statediff
 
 import (
-	"errors"
-	"github.com/ethereum/go-ethereum/statediff/ipfs"
+	"os"
+	"encoding/csv"
+	"time"
+	"strconv"
+	"strings"
 )
 
 type Publisher interface {
 	PublishStateDiff(sd *StateDiff) (string, error)
 }
 
 type publisher struct {
-	ipfs.DagPutter
-	Config
+	Config Config
 }
 
-func NewPublisher(config Config) (*publisher, error) {
-	adder, err := ipfs.NewAdder(config.Path)
-	if err != nil {
-		return nil, err
+var (
+	Headers = []string{
+		"blockNumber", "blockHash", "accountAction",
+		"code", "codeHash",
+		"oldNonceValue", "newNonceValue",
+		"oldBalanceValue", "newBalanceValue",
+		"oldContractRoot", "newContractRoot",
+		"storageDiffPaths",
 	}
 
+	timeStampFormat = "20060102150405.00000"
+	deletedAccountAction = "deleted"
+	createdAccountAction = "created"
+	updatedAccountAction = "updated"
+)
+
+func NewPublisher(config Config) (*publisher, error) {
 	return &publisher{
-		DagPutter: ipfs.NewDagPutter(adder),
 		Config: config,
 	}, nil
 }
 
 func (p *publisher) PublishStateDiff(sd *StateDiff) (string, error) {
-	switch p.Mode {
-	case IPLD:
-		cidStr, err := p.DagPut(sd)
+	switch p.Config.Mode {
+	case CSV:
+		return "", p.publishStateDiffToCSV(*sd)
+	default:
+		return "", p.publishStateDiffToCSV(*sd)
+	}
+}
+
+func (p *publisher) publishStateDiffToCSV(sd StateDiff) error {
+	now := time.Now()
+	timeStamp := now.Format(timeStampFormat)
+	filePath := p.Config.Path + timeStamp + ".csv"
+	file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return err
+	}
+	defer file.Close()
+
+	writer := csv.NewWriter(file)
+	defer writer.Flush()
+
+	var data [][]string
+	data = append(data, Headers)
+	for _, row := range accumulateCreatedAccountRows(sd) {
+		data = append(data, row)
+	}
+	for _, row := range accumulateUpdatedAccountRows(sd) {
+		data = append(data, row)
+	}
+
+	for _, row := range accumulateDeletedAccountRows(sd) {
+		data = append(data, row)
+	}
+
+	for _, value := range data{
+		err := writer.Write(value)
 		if err != nil {
-			return "", err
+			return err
 		}
+	}
 
-		return cidStr, err
-	case LDB:
-	case SQL:
-	default:
+	return nil
+}
+
+func accumulateUpdatedAccountRows(sd StateDiff) [][]string {
+	var updatedAccountRows [][]string
+	for _, accountDiff := range sd.UpdatedAccounts {
+		formattedAccountData := formatAccountDiffIncremental(accountDiff, sd, updatedAccountAction)
+
+		updatedAccountRows = append(updatedAccountRows, formattedAccountData)
+	}
+
+	return updatedAccountRows
+}
+
+func accumulateDeletedAccountRows(sd StateDiff) [][]string {
+	var deletedAccountRows [][]string
+	for _, accountDiff := range sd.DeletedAccounts {
+		formattedAccountData := formatAccountDiffEventual(accountDiff, sd, deletedAccountAction)
+
+		deletedAccountRows = append(deletedAccountRows, formattedAccountData)
+	}
+
+	return deletedAccountRows
+}
+
+func accumulateCreatedAccountRows(sd StateDiff) [][]string {
+	var createdAccountRows [][]string
+	for _, accountDiff := range sd.CreatedAccounts {
+		formattedAccountData := formatAccountDiffEventual(accountDiff, sd, createdAccountAction)
+
+		createdAccountRows = append(createdAccountRows, formattedAccountData)
 	}
 
-	return "", errors.New("state diff publisher: unhandled publishing mode")
-}
+	return createdAccountRows
+}
+
+func formatAccountDiffEventual(accountDiff AccountDiffEventual, sd StateDiff, accountAction string) []string {
+	oldContractRoot := accountDiff.ContractRoot.OldValue
+	newContractRoot := accountDiff.ContractRoot.NewValue
+	var storageDiffPaths []string
+	for k := range accountDiff.Storage {
+		storageDiffPaths = append(storageDiffPaths, k)
+	}
+	formattedAccountData := []string{
+		strconv.FormatInt(sd.BlockNumber, 10),
+		sd.BlockHash.String(),
+		accountAction,
+		string(accountDiff.Code),
+		accountDiff.CodeHash,
+		strconv.FormatUint(*accountDiff.Nonce.OldValue, 10),
+		strconv.FormatUint(*accountDiff.Nonce.NewValue, 10),
+		accountDiff.Balance.OldValue.String(),
+		accountDiff.Balance.NewValue.String(),
+		*oldContractRoot,
+		*newContractRoot,
+		strings.Join(storageDiffPaths, ","),
+	}
+	return formattedAccountData
+}
+
+func formatAccountDiffIncremental(accountDiff AccountDiffIncremental, sd StateDiff, accountAction string) []string {
+	oldContractRoot := accountDiff.ContractRoot.OldValue
+	newContractRoot := accountDiff.ContractRoot.NewValue
+	var storageDiffPaths []string
+	for k := range accountDiff.Storage {
+		storageDiffPaths = append(storageDiffPaths, k)
+	}
+	formattedAccountData := []string{
+		strconv.FormatInt(sd.BlockNumber, 10),
+		sd.BlockHash.String(),
+		accountAction,
+		"",
+		accountDiff.CodeHash,
+		strconv.FormatUint(*accountDiff.Nonce.OldValue, 10),
+		strconv.FormatUint(*accountDiff.Nonce.NewValue, 10),
+		accountDiff.Balance.OldValue.String(),
+		accountDiff.Balance.NewValue.String(),
+		*oldContractRoot,
+		*newContractRoot,
+		strings.Join(storageDiffPaths, ","),
+	}
+	return formattedAccountData
+}
+