Refactor: Decouple knownGaps and Indexer

This commit decouples knownGaps and Indexer. All knownGaps logic is in its own file. This makes testing and maintainability easier.

We have also removed all efforts to check the `lastProcessedblock` - This is because we won't ever run into this issue (hyptothetically), because geth won't let it happen.

Author: Abdul Rabbani

.gitignore

@@ -56,3 +56,4 @@ related-repositories/hive/**
 related-repositories/ipld-eth-db/**
 statediff/indexer/database/sql/statediffing_test_file.sql
 statediff/statediffing_test_file.sql
+statediff/known_gaps.sql

cmd/geth/config.go

@@ -177,7 +177,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 
 	if ctx.GlobalBool(utils.StateDiffFlag.Name) {
 		var indexerConfig interfaces.Config
-		var fileConfig interfaces.Config
 		var clientName, nodeID string
 		if ctx.GlobalIsSet(utils.StateDiffWritingFlag.Name) {
 			clientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name)
@@ -192,15 +191,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 			if err != nil {
 				utils.Fatalf("%v", err)
 			}
-
-			if dbType != shared.FILE {
-				fileConfig = file.Config{
-					FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name),
-				}
-			} else {
-				fileConfig = nil
-			}
-
 			switch dbType {
 			case shared.FILE:
 				indexerConfig = file.Config{
@@ -262,14 +252,14 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 			}
 		}
 		p := statediff.Config{
-			IndexerConfig:   indexerConfig,
-			FileConfig:      fileConfig,
-			ID:              nodeID,
-			ClientName:      clientName,
-			Context:         context.Background(),
-			EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
-			NumWorkers:      ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
-			WaitForSync:     ctx.GlobalBool(utils.StateDiffWaitForSync.Name),
+			IndexerConfig:     indexerConfig,
+			KnownGapsFilePath: ctx.GlobalString(utils.StateDiffKnownGapsFilePath.Name),
+			ID:                nodeID,
+			ClientName:        clientName,
+			Context:           context.Background(),
+			EnableWriteLoop:   ctx.GlobalBool(utils.StateDiffWritingFlag.Name),
+			NumWorkers:        ctx.GlobalUint(utils.StateDiffWorkersFlag.Name),
+			WaitForSync:       ctx.GlobalBool(utils.StateDiffWaitForSync.Name),
 		}
 		utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p, backend)
 	}

cmd/geth/main.go

@@ -176,6 +176,7 @@ var (
 		utils.StateDiffWritingFlag,
 		utils.StateDiffWorkersFlag,
 		utils.StateDiffFilePath,
+		utils.StateDiffKnownGapsFilePath,
 		utils.StateDiffWaitForSync,
 		configFileFlag,
 	}

cmd/geth/usage.go

@@ -246,6 +246,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.StateDiffWritingFlag,
 			utils.StateDiffWorkersFlag,
 			utils.StateDiffFilePath,
+			utils.StateDiffKnownGapsFilePath,
 			utils.StateDiffWaitForSync,
 		},
 	},

cmd/utils/flags.go

@@ -867,6 +867,11 @@ var (
 		Name:  "statediff.file.path",
 		Usage: "Full path (including filename) to write statediff data out to when operating in file mode",
 	}
+	StateDiffKnownGapsFilePath = cli.StringFlag{
+		Name:  "statediff.knowngapsfile.path",
+		Usage: "Full path (including filename) to write knownGaps statements when the DB is unavailable.",
+		Value: "./known_gaps.sql",
+	}
 	StateDiffDBClientNameFlag = cli.StringFlag{
 		Name:  "statediff.db.clientname",
 		Usage: "Client name to use when writing state diffs to database",

statediff/config.go

@@ -26,10 +26,10 @@ import (
 
 // Config contains instantiation parameters for the state diffing service
 type Config struct {
-	// The configuration used for the primary stateDiff Indexer
+	// The configuration used for the stateDiff Indexer
 	IndexerConfig interfaces.Config
-	// The configuration used for the file stateDiff Indexer. This indexer is used if the primary indexer fails and is not a FILE indexer.
-	FileConfig interfaces.Config
+	// The filepath to write knownGaps insert statements if we can't connect to the DB.
+	KnownGapsFilePath string
 	// A unique ID used for this service
 	ID string
 	// Name for the client this service is running

statediff/docs/KnownGaps.md

@@ -10,7 +10,6 @@ The known gaps table is updated when the following events occur:
 
 1. At start up we check the latest block from the `eth.headers_cid` table. We compare the first block that we are processing with the latest block from the DB. If they are not one unit of expectedDifference away from each other, add the gap between the two blocks.
 2. If there is any error in processing a block (db connection, deadlock, etc), add that block to the knownErrorBlocks slice, when the next block is successfully written, write this slice into the DB.
-3. If the last processed block is not one unit of expectedDifference away from the current block being processed. This can be due to any unknown or unhandled errors in geth.
 
 # Glossary
 

statediff/docs/database.md

@@ -0,0 +1,21 @@
+# Overview
+
+This document will go through some notes on the database component of the statediff service.
+
+# Components
+
+- Indexer: The indexer creates IPLD and DB models to insert to the Postgres DB. It performs the insert utilizing and atomic function.
+- Builder: The builder constructs the statediff object that needs to be inserted.
+- Known Gaps: Captures any gaps that might have occured and either writes them to the DB, local sql file, to prometeus, or a local error.
+
+# Making Code Changes
+
+## Adding a New Function to the Indexer
+
+If you want to implement a new feature for adding data to the database. Keep the following in mind:
+
+1. You need to handle `sql`, `file`, and `dump`.
+   1. `sql` - Contains the code needed to write directly to the `sql` db.
+   2. `file` - Contains all the code required to write the SQL statements to a file.
+   3. `dump` - Contains all the code for outputting events to the console.
+2. You will have to add it to the `interfaces.StateDiffIndexer` interface.

statediff/docs/diagrams/KnownGapsProcess.png

@@ -32,55 +32,50 @@ import (
 )
 
 // NewStateDiffIndexer creates and returns an implementation of the StateDiffIndexer interface.
-// You can specify the specific
-func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config, specificIndexer shared.DBType) (interfaces.StateDiffIndexer, error) {
-	var indexerToCreate shared.DBType
-	if specificIndexer == "" {
-		indexerToCreate = config.Type()
-	} else {
-		indexerToCreate = specificIndexer
-	}
-	log.Info("Indexer to create is", "indexer", indexerToCreate)
-	switch indexerToCreate {
+func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, nodeInfo node.Info, config interfaces.Config) (sql.Database, interfaces.StateDiffIndexer, error) {
+	switch config.Type() {
 	case shared.FILE:
-		log.Info("Creating a statediff indexer in SQL file writing mode")
+		log.Info("Starting statediff service in SQL file writing mode")
 		fc, ok := config.(file.Config)
 		if !ok {
-			return nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{})
+			return nil, nil, fmt.Errorf("file config is not the correct type: got %T, expected %T", config, file.Config{})
 		}
 		fc.NodeInfo = nodeInfo
-		return file.NewStateDiffIndexer(ctx, chainConfig, fc)
+		ind, err := file.NewStateDiffIndexer(ctx, chainConfig, fc)
+		return nil, ind, err
 	case shared.POSTGRES:
-		log.Info("Creating a statediff service in Postgres writing mode")
+		log.Info("Starting statediff service in Postgres writing mode")
 		pgc, ok := config.(postgres.Config)
 		if !ok {
-			return nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{})
+			return nil, nil, fmt.Errorf("postgres config is not the correct type: got %T, expected %T", config, postgres.Config{})
 		}
 		var err error
 		var driver sql.Driver
 		switch pgc.Driver {
 		case postgres.PGX:
 			driver, err = postgres.NewPGXDriver(ctx, pgc, nodeInfo)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		case postgres.SQLX:
 			driver, err = postgres.NewSQLXDriver(ctx, pgc, nodeInfo)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		default:
-			return nil, fmt.Errorf("unrecognized Postgres driver type: %s", pgc.Driver)
+			return nil, nil, fmt.Errorf("unrecognized Postgres driver type: %s", pgc.Driver)
 		}
-		return sql.NewStateDiffIndexer(ctx, chainConfig, postgres.NewPostgresDB(driver))
+		db := postgres.NewPostgresDB(driver)
+		ind, err := sql.NewStateDiffIndexer(ctx, chainConfig, db)
+		return db, ind, err
 	case shared.DUMP:
-		log.Info("Creating statediff indexer in data dump mode")
+		log.Info("Starting statediff service in data dump mode")
 		dumpc, ok := config.(dump.Config)
 		if !ok {
-			return nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{})
+			return nil, nil, fmt.Errorf("dump config is not the correct type: got %T, expected %T", config, dump.Config{})
 		}
-		return dump.NewStateDiffIndexer(chainConfig, dumpc), nil
+		return nil, dump.NewStateDiffIndexer(chainConfig, dumpc), nil
 	default:
-		return nil, fmt.Errorf("unrecognized database type: %s", indexerToCreate)
+		return nil, nil, fmt.Errorf("unrecognized database type: %s", config.Type())
 	}
 }