[IBC] Make the IBC Host a submodule with access to the bus (#868)

## Description

<!-- reviewpad:summarize:start -->
### Summary generated by Reviewpad on 13 Jul 23 10:41 UTC
This pull request includes several changes across multiple files. Here
is a summary of the changes:

1. In the file `ibc/handle_message_test.go`, the file was renamed to
`ibc/ibc_msg_mempool_test.go` with a similarity index of 85%.
2. In the same file, the imports were modified, and the `strings`
package was added.
3. The test function `TestHandleMessage_ErrorAlreadyInMempool` was
renamed to `TestEmitMessage_MessageAddedToLocalMempool`. Some code
within the renamed test function was also modified, including changes in
the preparation of test data and the addition of a transaction to the
mempool.
4. The test function `TestHandleMessage_BasicValidation_Message` was
renamed to `TestIBCMessage_BasicValidation_Message`.
5. The test function `TestHandleMessage_BasicValidation_Transaction` was
renamed to `TestIBCMessage_BasicValidation_Transaction`.
6. A new test function `TestHandleMessage_ErrorAlreadyInMempool` was
added to check for the error of having a duplicate transaction in the
mempool.
7. A new test function `TestHandleMessage_ErrorAlreadyCommitted` was
added to check for the error of having an already committed transaction.
8. The test function `TestHandleMessage_GetIndexedMessage` was modified
to include changes in the preparation of the environment.
9. The test function `TestHandleMessage_AddToMempool` was removed.

Additionally, other files such as `treestore_module.go`, `emitter.go`,
`submodule.go`, `config.validator4.json`, `persistence/ibc.go`,
`defaults.go`, `bus_module.go`, `ibc_store_module.go`,
`bulk_store_cache.go`, `ibc_host_module.go`, `config.validator4.json`,
`persistence/test/benchmark_state_test.go`, `p2p/README.md`,
`runtime/manager_test.go`, `shared/node.go`,
`persistence/test/manager_test.go`,
`shared/modules/bulk_store_cache.go`, `ibc.go`, `shared/node.go`,
`ics24.md`, `main_test.go` have also been modified.

Please let me know if you need more information or details about any
specific change.
<!-- reviewpad:summarize:end -->

## Issue

Fixes #854 

## Type of change

Please mark the relevant option(s):

- [x] New feature, functionality or library
- [ ] Bug fix
- [ ] Code health or cleanup
- [ ] Major breaking change
- [ ] Documentation
- [ ] Other <!-- add details here if it a different type of change -->

## List of changes

- Move IBC host into its own submodule
- Enable local ProvableStore instances to emit events to alter the IBC
state tree

## Testing

- [x] `make develop_test`; if any code changes were made
- [x] `make test_e2e` on [k8s
LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md);
if any code changes were made
- [x] `e2e-devnet-test` passes tests on
[DevNet](https://pocketnetwork.notion.site/How-to-DevNet-ff1598f27efe44c09f34e2aa0051f0dd);
if any code was changed
- [x] [Docker Compose
LocalNet](https://github.com/pokt-network/pocket/blob/main/docs/development/README.md);
if any major functionality was changed or introduced
- [x] [k8s
LocalNet](https://github.com/pokt-network/pocket/blob/main/build/localnet/README.md);
if any infrastructure or configuration changes were made


## Required Checklist

- [x] I have performed a self-review of my own code
- [x] I have commented my code, particularly in hard-to-understand areas
- [x] I have added, or updated, [`godoc` format
comments](https://go.dev/blog/godoc) on touched members (see:
[tip.golang.org/doc/comment](https://tip.golang.org/doc/comment))
- [x] I have tested my changes using the available tooling
- [ ] I have updated the corresponding CHANGELOG

### If Applicable Checklist

- [ ] I have updated the corresponding README(s); local and/or global
- [ ] I have added tests that prove my fix is effective or that my
feature works
- [ ] I have added, or updated,
[mermaid.js](https://mermaid-js.github.io) diagrams in the corresponding
README(s)
- [ ] I have added, or updated, documentation and
[mermaid.js](https://mermaid-js.github.io) diagrams in `shared/docs/*`
if I updated `shared/*`README(s)

Author: h5law

build/config/config.validator1.json

@@ -60,7 +60,9 @@
   },
   "ibc": {
     "enabled": true,
-    "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
-    "stores_dir": "/var/ibc"
+    "stores_dir": "/var/ibc",
+    "host": {
+        "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e"
+    }
   }
 }

build/config/config.validator2.json

@@ -53,7 +53,9 @@
   },
   "ibc": {
     "enabled": true,
-    "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
-    "stores_dir": "/var/ibc"
+    "stores_dir": "/var/ibc",
+    "host": {
+        "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e"
+    }
   }
 }

build/config/config.validator3.json

@@ -53,7 +53,9 @@
   },
   "ibc": {
     "enabled": true,
-    "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
-    "stores_dir": "/var/ibc"
+    "stores_dir": "/var/ibc",
+    "host": {
+        "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e"
+    }
   }
 }

build/config/config.validator4.json

@@ -53,7 +53,9 @@
   },
   "ibc": {
     "enabled": true,
-    "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e",
-    "stores_dir": "/var/ibc"
+    "stores_dir": "/var/ibc",
+    "host": {
+        "private_key": "0ca1a40ddecdab4f5b04fa0bfed1d235beaa2b8082e7554425607516f0862075dfe357de55649e6d2ce889acf15eb77e94ab3c5756fe46d3c7538d37f27f115e"
+    }
   }
 }

consensus/e2e_tests/utils_test.go

@@ -122,7 +122,8 @@ func CreateTestConsensusPocketNode(
 	telemetryMock := baseTelemetryMock(t, eventsChannel)
 	loggerMock := baseLoggerMock(t, eventsChannel)
 	rpcMock := baseRpcMock(t, eventsChannel)
-	ibcMock := ibcUtils.IBCMockWithHost(t, eventsChannel)
+	ibcMock, hostMock := ibcUtils.IBCMockWithHost(t, bus)
+	bus.RegisterModule(hostMock)
 
 	for _, module := range []modules.Module{
 		p2pMock,

ibc/docs/ics24.md

@@ -10,10 +10,11 @@
   - [IBC State Tree](#ibc-state-tree)
   - [Data Retrieval](#data-retrieval)
   - [IBC Messages](#ibc-messages)
-  - [IBC Message Handling](#ibc-message-handling)
+  - [IBC Message Broadcasting](#ibc-message-broadcasting)
   - [Mempool](#mempool)
   - [State Transition](#state-transition)
 - [Provable Stores](#provable-stores)
+- [Bulk Store Cacher](#bulk-store-cacher)
   - [Caching](#caching)
 
 ## Overview
@@ -26,11 +27,18 @@ As token transfers as defined in [ICS-20][ics20] work on a lock and mint pattern
 
 Only validators can be configured to be IBC hosts. If the IBC module, during its creation, detects the node is a validator (and the IBC `enabled` field in the config is `true`) it will automatically create a host.
 
+An example of the JSON configuration for the IBC module is as follows:
+
 ```json
 "ibc": {
-    "enabled": bool,
-    "private_key": string,
-    "stores_dir": string
+  "enabled": true,
+  "stores_dir": "/var/ibc",
+  "host": {
+    "private_key": "private key hex string",
+    "bulk_store_cacher": {
+      "max_height_stored": 5
+    }
+  }
 }
 ```
 
@@ -116,7 +124,7 @@ When attempting to generate a proof for a specific `key` in the IBC state tree t
 
 ### IBC Messages
 
-Hosts maintain uncommitted changes in a local ephemeral IBC store while messages propagate through the mempool.
+Hosts propagate any local changes they propose to the IBC store throughout the network's existing `Transaction` flow, and they handle them locally.
 
 These messages enable a variety of IBC related state changes such as creating light clients, opening connections, sending packets, etc... This is enabled by propagating `IBCMessage` types defined in [ibc/types/proto/messages.proto](../types/proto/messages.proto). This type acts as an enum representing two possible state transition events:
 
@@ -127,42 +135,51 @@ _Note: In both types described above the `key` field **must** already be prefixe
 
 When changes are made locally they are not committed to the IBC store itself but are instead used to create an `IBCMessage` which is broadcasted to the network. This is akin to a simple send transaction that has been propagated throughout the mempool but has not been committed to the on-chain state.
 
-### IBC Message Handling
+### IBC Message Broadcasting
 
-Upon a node receiving an `IBCMessage` from the event bus it will use the `HandleMessage()` method of the `IBCModule` to add this message to the transactions mempool via the following steps:
+Upon making a local change to the IBC store the host will:
 
-1. Wrap the `IBCMessage` within a `Transaction`
-2. Sign the `Transaction` using the `IBCModule`'s private key
-3. Broadcast the `Transaction` throughout the mempool
+1. Create the relevent `IBCMessage` type
+   - `UpdateIBCStore`: for creation/update events
+   - `PruneIBCStore`: for deletion events
+2. Wrap the `IBCMessage` within a `Transaction`
+3. Sign the `Transaction` using the IBC Host's private key
+4. Add the `Transaction` to the local mempool
+5. Broadcast the `Transaction` via the `P2P` module for the other nodes to include it in their mempools
 
 ```mermaid
-graph LR
-  subgraph Bus
-    A[Events]
+graph TD
+  subgraph Update
+    A[Key]
+    B[Value]
+  end
+  subgraph UpdateIBCStore
+    C[Signer]
+    D[Key]
+    E[Value]
   end
-  subgraph I[IBC Host]
-    I1["HandleMessage(Message)"]
+  subgraph IBCMessage
+    F[Event]
   end
-  subgraph Handler
-    H1["ConvertIBCMessageToTransaction(IBCMessage)"]
-    subgraph Transaction
-      T1["coreTypes.Transaction{Msg: IBCMessage}"]
-    end
-    H2["SignTransaction(Transaction)"]
+  subgraph Transaction
+    G[Msg]
+    H[Nonce]
+    I[Signature]
   end
   subgraph Mempool
-    M1["ValidateTransaction(Transaction)"]
-    M2["AddToMempool(Transaction)"]
+    J[Txs]
+  end
+  subgraph P2P
+    K[Broadcast]
   end
-  Bus--Message-->I
-  I--IBCMessage-->Handler
-  H1--IBCMessage-->Transaction
-  Transaction--Transaction-->H2
-  Handler--Transaction-->Mempool
-  M1--Transaction-->M2
+  Update -- "Update(Key, Value)" --> UpdateIBCStore
+  UpdateIBCStore -- UpdateIBCStore --> IBCMessage
+  IBCMessage -- Message --> Transaction
+  Transaction -- Tx --> Mempool
+  Transaction -- Tx --> P2P
 ```
 
-See: [ibc/module.go](../module.go) for the specific implementation details.
+See: [emitter.go](../store/emitter.go) for the specific implementation details.
 
 ### Mempool
 
@@ -177,15 +194,29 @@ See: [PROTOCOL_STATE_HASH.md](../../persistence/docs/PROTOCOL_STATE_HASH.md#ibc-
 
 ## Provable Stores
 
-The `ProvableStore` interface defined in [shared/modules/ibc_module.go](../../shared/modules/ibc_module.go) is implemented by the [`provableStore`](../store/provable_store.go) type and managed by the [`StoreManager`](../store/store_manager.go).
+The `ProvableStore` interface defined in [ibc_store_module.go](../../shared/modules/ibc_store_module.go) is implemented by the [`provableStore`](../store/provable_store.go) type and managed by the [`BulkStoreCacher`](../store/bulk_store_cache.go).
 
-The provable stores are each assigned a `prefix`. This represents the specific sub-store that they are able to access and interact with in the IBC state tree. When doing any operation `get`/`set`/`delete` the `prefix` is applied to the `key` provided to generate the `CommitmentPath` to the element in the IBC state tree.
+The provable stores are each assigned a `prefix`. This represents the specific sub-store that they are able to access and interact with in the IBC state tree. When doing any operation `get`/`set`/`delete` the `prefix` is applied to the `key` provided (if not already present) to generate the `CommitmentPath` to the element in the IBC state tree. The `CommitmentPrefix` for any provable store can be obtained through the following method:
+
+```go
+type ProvableStore interface {
+  ...
+  GetCommitmentPrefix() CommitmentPrefix
+  ...
+}
+```
 
 The provable stores do not directly interface with the IBC state tree but instead utliise the `peristence` layer to query the data locally. This allows for the efficient querying of the IBC store instead of having to query the IBC state tree directly. Any changes made by the `ProvableStore` instance are broadcasted to the network for inclusion in the next block, being stored in their mempools.
 
+## Bulk Store Cacher
+
+The [`BulkStoreCacher`](../store/bulk_store_cache.go) manages all the different provable store instances. It keeps an in-memory map of the current active provable stores, and also manages their caches in bulk.
+
 ### Caching
 
-Every local change made to the IBC store (`update`/`delete`) is stored in an in-memory cache. These caches are written to disk by the [`StoreManager`](../store/store_manager.go).
+Every local change made to the IBC store (`update`/`delete`) is stored in an in-memory cache. These caches are written to disk (i.e. flushed) by the `BulkStoreCacher`, upon the receipt of a new height event from consensus. This means that the cache for any given height is always written to disk all at once.
+
+The `BulkStoreCacher` also keeps track of the maximum height stored in the cache. The `max_height_stored` value (default `5`) is used when a new height event comes in to prune old entries in the cache stored on disk in a "circular buffer" like fashion.
 
 In the event of a node failure, or local changes not being propagated correctly. Any changes stored in the cache can be "replayed" by the node and broadcasted to the network for inclusion in the next block.
 

ibc/host.go

@@ -0,0 +1,85 @@
+package host
+
+import (
+	"errors"
+	"time"
+
+	"github.com/pokt-network/pocket/ibc/store"
+	"github.com/pokt-network/pocket/runtime/configs"
+	coreTypes "github.com/pokt-network/pocket/shared/core/types"
+	"github.com/pokt-network/pocket/shared/modules"
+	"github.com/pokt-network/pocket/shared/modules/base_modules"
+)
+
+var _ modules.IBCHostSubmodule = &ibcHost{}
+
+type ibcHost struct {
+	base_modules.IntegrableModule
+
+	cfg       *configs.IBCHostConfig
+	logger    *modules.Logger
+	storesDir string
+}
+
+func Create(bus modules.Bus, config *configs.IBCHostConfig, options ...modules.IBCHostOption) (modules.IBCHostSubmodule, error) {
+	return new(ibcHost).Create(bus, config, options...)
+}
+
+// WithLogger assigns a logger for the IBC host
+func WithLogger(logger *modules.Logger) modules.IBCHostOption {
+	return func(m modules.IBCHostSubmodule) {
+		if mod, ok := m.(*ibcHost); ok {
+			mod.logger = logger
+		}
+	}
+}
+
+// WithStoresDir assigns a stores directory for the IBC host
+func WithStoresDir(dir string) modules.IBCHostOption {
+	return func(m modules.IBCHostSubmodule) {
+		if mod, ok := m.(*ibcHost); ok {
+			mod.storesDir = dir
+		}
+	}
+}
+
+func (*ibcHost) Create(bus modules.Bus, config *configs.IBCHostConfig, options ...modules.IBCHostOption) (modules.IBCHostSubmodule, error) {
+	h := &ibcHost{
+		cfg: config,
+	}
+	for _, option := range options {
+		option(h)
+	}
+	h.logger.Info().Msg("🛰️ Creating IBC host 🛰️")
+	bus.RegisterModule(h)
+	_, err := store.Create(h.GetBus(),
+		h.cfg.BulkStoreCacher,
+		store.WithLogger(h.logger),
+		store.WithStoresDir(h.storesDir),
+		store.WithPrivateKey(h.cfg.PrivateKey),
+	)
+	if err != nil {
+		return nil, err
+	}
+	return h, nil
+}
+
+func (h *ibcHost) GetModuleName() string { return modules.IBCHostSubmoduleName }
+
+// GetTimestamp returns the current unix timestamp
+func (h *ibcHost) GetTimestamp() uint64 {
+	return uint64(time.Now().Unix())
+}
+
+// GetProvableStore returns an instance of a provable store with the given name with the
+// CommitmentPrefix set to []byte(name). The store is created if it does not exist.
+//
+// Any changes made using the store are handled locally and propagated through the bus,
+// added to all nodes' mempools ready for inclusion in the next block to transition the IBC store state tree.
+// Any operations will ensure the CommitmentPrefix is prepended to the key if not present already.
+func (h *ibcHost) GetProvableStore(name string) (modules.ProvableStore, error) {
+	if err := h.GetBus().GetBulkStoreCacher().AddStore(name); err != nil && !errors.Is(err, coreTypes.ErrIBCStoreAlreadyExists(name)) {
+		return nil, err
+	}
+	return h.GetBus().GetBulkStoreCacher().GetStore(name)
+}