[IBC] Make the IBC Host a submodule with access to the bus

ibc/docs/ics24.md

@@ -147,6 +147,38 @@ Upon making a local change to the IBC store the host will:
 4. Add the `Transaction` to the local mempool
 5. Broadcast the `Transaction` via the `P2P` module for the other nodes to include it in their mempools
 
+```mermaid
+graph TD
+  subgraph Update
+    A[Key]
+    B[Value]
+  end
+  subgraph UpdateIBCStore
+    C[Signer]
+    D[Key]
+    E[Value]
+  end
+  subgraph IBCMessage
+    F[Event]
+  end
+  subgraph Transaction
+    G[Msg]
+    H[Nonce]
+    I[Signature]
+  end
+  subgraph Mempool
+    J[Txs]
+  end
+  subgraph P2P
+    K[Broadcast]
+  end
+  Update -- "Update(Key, Value)" --> UpdateIBCStore
+  UpdateIBCStore -- UpdateIBCStore --> IBCMessage
+  IBCMessage -- Message --> Transaction
+  Transaction -- Tx --> Mempool
+  Transaction -- Tx --> P2P
+```
+
 See: [emitter.go](../store/emitter.go) for the specific implementation details.
 
 ### Mempool

ibc/ibc_handle_event_test.go

@@ -94,7 +94,7 @@ func TestHandleEvent_FlushCaches(t *testing.T) {
 	require.NoError(t, cache.Stop())
 }
 
-// using MaxStoredHeight = 3
+// using MaxHeightCached = 3
 func TestHandleEvent_PruneCaches(t *testing.T) {
 	mgr, _, _, _, ibcMod := prepareEnvironment(t, 1, 0, 0, 0)
 	tmpDir := mgr.GetConfig().IBC.StoresDir + "/caches"
@@ -125,49 +125,49 @@ func TestHandleEvent_PruneCaches(t *testing.T) {
 	}
 
 	testCases := []struct {
-		name            string
-		heights         []uint64
-		expectedHeights []uint64
-		length          int
+		name                  string
+		heightsStored         []uint64 // the different heights where entries are written to the cache
+		expectedHeightsCached []uint64 // the different heights expected in the cache after pruning
+		cacheLength           int      // the length of the cache after pruning
 	}{
 		{
-			name:            "No pruning after single height increase",
-			heights:         []uint64{1, 2},
-			expectedHeights: []uint64{1},
-			length:          4,
+			name:                  "No pruning after single height increase",
+			heightsStored:         []uint64{1, 2},
+			expectedHeightsCached: []uint64{1},
+			cacheLength:           4,
 		},
 		{
-			name:            "No pruning after two height increase",
-			heights:         []uint64{1, 2, 3},
-			expectedHeights: []uint64{1, 2},
-			length:          8,
+			name:                  "No pruning after two height increase",
+			heightsStored:         []uint64{1, 2, 3},
+			expectedHeightsCached: []uint64{1, 2},
+			cacheLength:           8,
 		},
 		{
-			name:            "No pruning at max height stored = 3",
-			heights:         []uint64{1, 2, 3, 4},
-			expectedHeights: []uint64{1, 2, 3},
-			length:          12,
+			name:                  "No pruning at max height stored = 3",
+			heightsStored:         []uint64{1, 2, 3, 4},
+			expectedHeightsCached: []uint64{1, 2, 3},
+			cacheLength:           12,
 		},
 		{
-			name:            "Pruning after 4 height increase",
-			heights:         []uint64{1, 2, 3, 4, 5},
-			expectedHeights: []uint64{2, 3, 4},
-			length:          12,
+			name:                  "Pruning after 4 height increase",
+			heightsStored:         []uint64{1, 2, 3, 4, 5},
+			expectedHeightsCached: []uint64{2, 3, 4},
+			cacheLength:           12,
 		},
 		{
-			name:            "Pruning after 5 height increase",
-			heights:         []uint64{1, 2, 3, 4, 5, 6},
-			expectedHeights: []uint64{3, 4, 5},
-			length:          12,
+			name:                  "Pruning after 5 height increase",
+			heightsStored:         []uint64{1, 2, 3, 4, 5, 6},
+			expectedHeightsCached: []uint64{3, 4, 5},
+			cacheLength:           12,
 		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			// set height
-			publishNewHeightEvent(t, ibcMod.GetBus(), tc.heights[0])
+			publishNewHeightEvent(t, ibcMod.GetBus(), tc.heightsStored[0])
 
-			for _, height := range tc.heights[1:] {
+			for _, height := range tc.heightsStored[1:] {
 				for _, kv := range kvs {
 					if kv.value != nil {
 						require.NoError(t, store.Set(kv.key, kv.value))
@@ -186,13 +186,14 @@ func TestHandleEvent_PruneCaches(t *testing.T) {
 			require.NoError(t, err)
 			keys, values, err := cache.GetAll([]byte{}, false)
 			require.NoError(t, err)
-			require.Len(t, keys, tc.length)
-			require.Len(t, values, tc.length)
-
-			// iterate over the expected heights and check the keys and values are correct
-			for i, height := range tc.expectedHeights {
-				for j, key := range keys[i*4 : (i+1)*4] {
-					require.Equal(t, string(key), prepareCacheKey(height, kvs[j%4].key))
+			require.Len(t, keys, tc.cacheLength)
+			require.Len(t, values, tc.cacheLength)
+
+			// iterate over all the keys in batches of 4 (for each expected height) confirming the expected height
+			// is the same as the height stored in the cache for each batch
+			for i, height := range tc.expectedHeightsCached {
+				for j, key := range keys[i*4 : (i+1)*4] { // split keys into batches of 4 per height expected
+					require.Equal(t, string(key), prepareCacheKey(height, kvs[j%4].key)) // validate the expected height is the actual height
 					if kvs[j%4].value == nil {
 						require.Equal(t, values[j], []byte{})
 						continue
@@ -201,29 +202,6 @@ func TestHandleEvent_PruneCaches(t *testing.T) {
 				}
 			}
 
-			err = cache.ClearAll()
-			require.NoError(t, err)
-
-			newKeys, newValues, err := cache.GetAll([]byte{}, false)
-			require.NoError(t, err)
-			require.Len(t, newKeys, 0)
-			require.Len(t, newValues, 0)
-
-			require.NoError(t, cache.Stop())
-
-			// flush the cache
-			err = ibcHost.GetBus().GetBulkStoreCacher().FlushAllEntries()
-			require.NoError(t, err)
-
-			cache, err = kvstore.NewKVStore(tmpDir)
-			require.NoError(t, err)
-
-			// check in memory cache was cleared (ie nothing flushed)
-			newKeys, newValues, err = cache.GetAll([]byte{}, false)
-			require.NoError(t, err)
-			require.Len(t, newKeys, 0)
-			require.Len(t, newValues, 0)
-
 			require.NoError(t, cache.Stop())
 		})
 	}

ibc/main_test.go

@@ -198,7 +198,7 @@ func newTestRuntimeConfig(
 			Host: &configs.IBCHostConfig{
 				PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
 				BulkStoreCacher: &configs.BulkStoreCacherConfig{
-					MaxHeightStored: 3,
+					MaxHeightCached: 3,
 				},
 			},
 		},

ibc/module.go

@@ -99,10 +99,10 @@ func (m *ibcModule) HandleEvent(event *anypb.Any) error {
 			return err
 		}
 		// Prune old cache entries
-		if currHeight <= m.cfg.Host.BulkStoreCacher.MaxHeightStored {
+		if currHeight <= m.cfg.Host.BulkStoreCacher.MaxHeightCached {
 			break
 		}
-		pruneHeight := currHeight - m.cfg.Host.BulkStoreCacher.MaxHeightStored
+		pruneHeight := currHeight - m.cfg.Host.BulkStoreCacher.MaxHeightCached
 		if err := bsc.PruneCaches(pruneHeight); err != nil {
 			return err
 		}

runtime/configs/config.go

@@ -166,7 +166,7 @@ func NewDefaultConfig(options ...func(*Config)) *Config {
 			StoresDir: defaults.DefaultIBCStoresDir,
 			Host: &IBCHostConfig{
 				BulkStoreCacher: &BulkStoreCacherConfig{
-					MaxHeightStored: defaults.DefaultIBCCacheMaxHeightStored,
+					MaxHeightCached: defaults.DefaultIBCCacheMaxHeightCached,
 				},
 			},
 		},

runtime/configs/proto/ibc_config.proto

@@ -23,5 +23,6 @@ message IBCHostConfig {
 
 message BulkStoreCacherConfig {
     // INVESTIGATE: Should we cap the cache size by size (bytes) instead of heights which could, theoretically, grow unbounded
-    uint64 max_height_stored = 1;      // maximum number of heights to be stored in the cache before pruning
+    // For example, if num_heights_cached=5, then at height=100, we would cache heights [96,100], and at height=101, we would drop 96 and add 101
+    uint64 max_height_cached = 1;      // maximum number of heights to be stored in the cache before pruning
 }

runtime/defaults/defaults.go

@@ -75,7 +75,7 @@ var (
 	// ibc
 	DefaultIBCEnabled              = false
 	DefaultIBCStoresDir            = "/var/ibc"
-	DefaultIBCCacheMaxHeightStored = uint64(5)
+	DefaultIBCCacheMaxHeightCached = uint64(5)
 )
 
 var (

runtime/manager_test.go

@@ -1826,7 +1826,7 @@ func TestNewManagerFromReaders(t *testing.T) {
 						Host: &configs.IBCHostConfig{
 							PrivateKey: "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
 							BulkStoreCacher: &configs.BulkStoreCacherConfig{
-								MaxHeightStored: defaults.DefaultIBCCacheMaxHeightStored,
+								MaxHeightCached: defaults.DefaultIBCCacheMaxHeightCached,
 							},
 						},
 					},