Problem: ibc incentivization is not integrated (#647)

* support hermes 1.x

* Problem: ibc incentivization is not integrated

Closes: #642
Solution:
- integration ibcfee middleware

* fix build

* point back pystarport

* fix ibc config

* add ibcfee store to upgrade handler

* update chain-main to fix integration test

* Update app/app.go

* Update CHANGELOG.md

* ibc incentivized integration test

* fix fee middleware wiring

* run test cases for both incentivized and non-incentivized channels

Co-authored-by: mmsqe <mavis@crypto.com>

Author: yihuang

CHANGELOG.md

@@ -8,6 +8,7 @@
 - [cronos#600](https://github.com/crypto-org-chain/cronos/pull/600) Implement bidirectional token mapping.
 - [cronos#611](https://github.com/crypto-org-chain/cronos/pull/611) Fix mistake on acknowledgement error in ibc middleware.
 - [cronos#627](https://github.com/crypto-org-chain/cronos/pull/627) Upgrade gravity bridge module with security enhancements
+- [cronos#647](https://github.com/crypto-org-chain/cronos/pull/647) Integrate ibc fee middleware.
 
 ### Bug Fixes
 

app/app.go

@@ -90,6 +90,9 @@ import (
 	icacontrollerkeeper "github.com/cosmos/ibc-go/v5/modules/apps/27-interchain-accounts/controller/keeper"
 	icacontrollertypes "github.com/cosmos/ibc-go/v5/modules/apps/27-interchain-accounts/controller/types"
 	icatypes "github.com/cosmos/ibc-go/v5/modules/apps/27-interchain-accounts/types"
+	ibcfee "github.com/cosmos/ibc-go/v5/modules/apps/29-fee"
+	ibcfeekeeper "github.com/cosmos/ibc-go/v5/modules/apps/29-fee/keeper"
+	ibcfeetypes "github.com/cosmos/ibc-go/v5/modules/apps/29-fee/types"
 
 	appparams "github.com/cosmos/cosmos-sdk/simapp/params"
 	"github.com/cosmos/ibc-go/v5/modules/apps/transfer"
@@ -186,6 +189,7 @@ var (
 		stakingtypes.NotBondedPoolName: {authtypes.Burner, authtypes.Staking},
 		govtypes.ModuleName:            {authtypes.Burner},
 		ibctransfertypes.ModuleName:    {authtypes.Minter, authtypes.Burner},
+		ibcfeetypes.ModuleName:         nil,
 		icatypes.ModuleName:            nil,
 		evmtypes.ModuleName:            {authtypes.Minter, authtypes.Burner}, // used for secure addition and subtraction of balance using module account
 		gravitytypes.ModuleName:        {authtypes.Minter, authtypes.Burner},
@@ -229,6 +233,7 @@ func GenModuleBasics(experimental bool) module.BasicManager {
 		vesting.AppModuleBasic{},
 		ica.AppModuleBasic{},
 		icactlmodule.AppModuleBasic{},
+		ibcfee.AppModuleBasic{},
 		evm.AppModuleBasic{},
 		feemarket.AppModuleBasic{},
 		// this line is used by starport scaffolding # stargate/app/moduleBasic
@@ -270,6 +275,7 @@ type App struct {
 	UpgradeKeeper       upgradekeeper.Keeper
 	ParamsKeeper        paramskeeper.Keeper
 	IBCKeeper           *ibckeeper.Keeper // IBC Keeper must be a pointer in the app, so we can SetRouter on it correctly
+	IBCFeeKeeper        ibcfeekeeper.Keeper
 	ICAControllerKeeper icacontrollerkeeper.Keeper
 	ICAAuthKeeper       icactlmodulekeeper.Keeper
 	EvidenceKeeper      evidencekeeper.Keeper
@@ -335,6 +341,7 @@ func New(
 		// ibc keys
 		ibchost.StoreKey, ibctransfertypes.StoreKey,
 		icacontrollertypes.StoreKey,
+		ibcfeetypes.StoreKey,
 		// ethermint keys
 		evmtypes.StoreKey, feemarkettypes.StoreKey,
 		// this line is used by starport scaffolding # stargate/app/storeKey
@@ -416,10 +423,18 @@ func New(
 		appCodec, keys[ibchost.StoreKey], app.GetSubspace(ibchost.ModuleName), stakingKeeper, app.UpgradeKeeper, scopedIBCKeeper,
 	)
 
+	// IBC Fee Module keeper
+	app.IBCFeeKeeper = ibcfeekeeper.NewKeeper(
+		appCodec, keys[ibcfeetypes.StoreKey], app.GetSubspace(ibcfeetypes.ModuleName),
+		app.IBCKeeper.ChannelKeeper, // more middlewares can be added in future
+		app.IBCKeeper.ChannelKeeper,
+		&app.IBCKeeper.PortKeeper, app.AccountKeeper, app.BankKeeper,
+	)
+
 	// Create Transfer Keepers
 	app.TransferKeeper = ibctransferkeeper.NewKeeper(
 		appCodec, keys[ibctransfertypes.StoreKey], app.GetSubspace(ibctransfertypes.ModuleName),
-		app.IBCKeeper.ChannelKeeper, app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
+		app.IBCFeeKeeper, app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
 		app.AccountKeeper, app.BankKeeper, scopedTransferKeeper,
 	)
 
@@ -494,21 +509,28 @@ func New(
 
 	// set the middleware
 	transferModule := transfer.NewAppModule(app.TransferKeeper)
-	transferIBCModule := transfer.NewIBCModule(app.TransferKeeper)
-	transferWithConversionIBCModule := middleware.NewIBCConversionModule(transferIBCModule, app.CronosKeeper)
+	feeModule := ibcfee.NewAppModule(app.IBCFeeKeeper)
+
+	var transferStack porttypes.IBCModule
+	transferStack = transfer.NewIBCModule(app.TransferKeeper)
+	transferStack = middleware.NewIBCConversionModule(transferStack, app.CronosKeeper)
+	transferStack = ibcfee.NewIBCMiddleware(transferStack, app.IBCFeeKeeper)
+
 	app.ICAControllerKeeper = icacontrollerkeeper.NewKeeper(
 		appCodec, keys[icacontrollertypes.StoreKey], app.GetSubspace(icacontrollertypes.SubModuleName),
-		app.IBCKeeper.ChannelKeeper, // may be replaced with middleware such as ics29 fee
-		app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
+		app.IBCFeeKeeper, app.IBCKeeper.ChannelKeeper, &app.IBCKeeper.PortKeeper,
 		scopedICAControllerKeeper, app.MsgServiceRouter(),
 	)
 	icaModule := ica.NewAppModule(&app.ICAControllerKeeper, nil)
 
 	app.ICAAuthKeeper = *icactlmodulekeeper.NewKeeper(appCodec,
 		app.GetSubspace(icactlmoduletypes.ModuleName), app.ICAControllerKeeper, scopedICAAuthKeeper)
-	icaAuthModule := icactlmodule.NewAppModule(appCodec, app.ICAAuthKeeper)
-	icaAuthIBCModule := icactlmodule.NewIBCModule(app.ICAAuthKeeper)
-	icaControllerIBCModule := icacontroller.NewIBCMiddleware(icaAuthIBCModule, app.ICAControllerKeeper)
+	icaCtlModule := icactlmodule.NewAppModule(appCodec, app.ICAAuthKeeper)
+
+	var icaControllerStack porttypes.IBCModule
+	icaControllerStack = icactlmodule.NewIBCModule(app.ICAAuthKeeper)
+	icaControllerStack = icacontroller.NewIBCMiddleware(icaControllerStack, app.ICAControllerKeeper)
+	icaControllerStack = ibcfee.NewIBCMiddleware(icaControllerStack, app.IBCFeeKeeper)
 
 	app.GovKeeper = govkeeper.NewKeeper(
 		appCodec, keys[govtypes.StoreKey], app.GetSubspace(govtypes.ModuleName), app.AccountKeeper, app.BankKeeper,
@@ -550,9 +572,9 @@ func New(
 
 	// Create static IBC router, add transfer route, then set and seal it
 	ibcRouter := porttypes.NewRouter()
-	ibcRouter.AddRoute(ibctransfertypes.ModuleName, transferWithConversionIBCModule)
-	ibcRouter.AddRoute(icacontrollertypes.SubModuleName, icaControllerIBCModule)
-	ibcRouter.AddRoute(icactlmoduletypes.ModuleName, icaControllerIBCModule)
+	ibcRouter.AddRoute(ibctransfertypes.ModuleName, transferStack)
+	ibcRouter.AddRoute(icacontrollertypes.SubModuleName, icaControllerStack)
+	ibcRouter.AddRoute(icactlmoduletypes.ModuleName, icaControllerStack)
 	// this line is used by starport scaffolding # ibc/app/router
 	app.IBCKeeper.SetRouter(ibcRouter)
 
@@ -588,7 +610,8 @@ func New(
 		ibc.NewAppModule(app.IBCKeeper),
 		transferModule,
 		icaModule,
-		icaAuthModule,
+		icaCtlModule,
+		feeModule,
 		evm.NewAppModule(app.EvmKeeper, app.AccountKeeper),
 		feemarket.NewAppModule(app.FeeMarketKeeper),
 		cronosModule,
@@ -607,6 +630,7 @@ func New(
 		ibctransfertypes.ModuleName,
 		icatypes.ModuleName,
 		icactlmoduletypes.ModuleName,
+		ibcfeetypes.ModuleName,
 		authtypes.ModuleName,
 		banktypes.ModuleName,
 		govtypes.ModuleName,
@@ -626,6 +650,7 @@ func New(
 		ibctransfertypes.ModuleName,
 		icatypes.ModuleName,
 		icactlmoduletypes.ModuleName,
+		ibcfeetypes.ModuleName,
 		capabilitytypes.ModuleName,
 		authtypes.ModuleName,
 		banktypes.ModuleName,
@@ -667,6 +692,7 @@ func New(
 		ibctransfertypes.ModuleName,
 		icatypes.ModuleName,
 		icactlmoduletypes.ModuleName,
+		ibcfeetypes.ModuleName,
 		authz.ModuleName,
 		feegrant.ModuleName,
 		paramstypes.ModuleName,

app/upgrades.go

@@ -8,6 +8,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/types/module"
 	upgradetypes "github.com/cosmos/cosmos-sdk/x/upgrade/types"
 	icacontrollertypes "github.com/cosmos/ibc-go/v5/modules/apps/27-interchain-accounts/controller/types"
+	ibcfeetypes "github.com/cosmos/ibc-go/v5/modules/apps/29-fee/types"
 	gravitytypes "github.com/peggyjv/gravity-bridge/module/v2/x/gravity/types"
 )
 
@@ -50,13 +51,22 @@ func (app *App) RegisterUpgradeHandlers(experimental bool) {
 		panic(fmt.Sprintf("failed to read upgrade info from disk %s", err))
 	}
 
-	if (upgradeInfo.Name == planName || (experimental && upgradeInfo.Name == gravityPlanName)) &&
-		!app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
-		storeUpgrades := storetypes.StoreUpgrades{
-			Added: []string{icacontrollertypes.StoreKey},
+	if !app.UpgradeKeeper.IsSkipHeight(upgradeInfo.Height) {
+		if upgradeInfo.Name == planName {
+			storeUpgrades := storetypes.StoreUpgrades{
+				Added: []string{icacontrollertypes.StoreKey, ibcfeetypes.StoreKey},
+			}
+
+			// configure store loader that checks if version == upgradeHeight and applies store upgrades
+			app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
 		}
+		if experimental && upgradeInfo.Name == gravityPlanName {
+			storeUpgrades := storetypes.StoreUpgrades{
+				Added: []string{icacontrollertypes.StoreKey},
+			}
 
-		// configure store loader that checks if version == upgradeHeight and applies store upgrades
-		app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
+			// configure store loader that checks if version == upgradeHeight and applies store upgrades
+			app.SetStoreLoader(upgradetypes.UpgradeStoreLoader(upgradeInfo.Height, &storeUpgrades))
+		}
 	}
 }

integration_tests/configs/ibc.jsonnet

@@ -5,6 +5,9 @@ config {
     'account-prefix': 'crc',
     'coin-type': 60,
     key_name: 'signer1',
+    accounts: super.accounts[:std.length(super.accounts) - 1] + [super.accounts[std.length(super.accounts) - 1] {
+      coins: super.coins + ',100000000000ibcfee',
+    }],
     'app-config'+: {
       'index-events': super['index-events'] + ['message.action'],
     },

integration_tests/cosmoscli.py

@@ -1308,3 +1308,58 @@ def create_vesting_account(self, to_address, amount, end_time, **kwargs):
                 **(default_kwargs | kwargs),
             )
         )
+
+    def register_counterparty_payee(
+        self, port_id, channel_id, relayer, counterparty_payee, **kwargs
+    ):
+        default_kwargs = {
+            "home": self.data_dir,
+        }
+        return json.loads(
+            self.raw(
+                "tx",
+                "ibc-fee",
+                "register-counterparty-payee",
+                port_id,
+                channel_id,
+                relayer,
+                counterparty_payee,
+                "-y",
+                **(default_kwargs | kwargs),
+            )
+        )
+
+    def register_payee(self, port_id, channel_id, relayer, payee, **kwargs):
+        default_kwargs = {
+            "home": self.data_dir,
+        }
+        return json.loads(
+            self.raw(
+                "tx",
+                "ibc-fee",
+                "register-payee",
+                port_id,
+                channel_id,
+                relayer,
+                payee,
+                "-y",
+                **(default_kwargs | kwargs),
+            )
+        )
+
+    def pay_packet_fee(self, port_id, channel_id, packet_seq, **kwargs):
+        default_kwargs = {
+            "home": self.data_dir,
+        }
+        return json.loads(
+            self.raw(
+                "tx",
+                "ibc-fee",
+                "pay-packet-fee",
+                port_id,
+                channel_id,
+                str(packet_seq),
+                "-y",
+                **(default_kwargs | kwargs),
+            )
+        )

integration_tests/ibc_utils.py

@@ -15,9 +15,10 @@ class IBCNetwork(NamedTuple):
     cronos: Cronos
     chainmain: Chainmain
     hermes: Hermes
+    incentivized: bool
 
 
-def prepare_network(tmp_path, file):
+def prepare_network(tmp_path, file, incentivized=True):
     file = f"configs/{file}.jsonnet"
     gen = setup_custom_cronos(tmp_path, 26700, Path(__file__).parent / file)
     cronos = next(gen)
@@ -26,6 +27,17 @@ def prepare_network(tmp_path, file):
     # wait for grpc ready
     wait_for_port(ports.grpc_port(chainmain.base_port(0)))  # chainmain grpc
     wait_for_port(ports.grpc_port(cronos.base_port(0)))  # cronos grpc
+
+    version = {"fee_version": "ics29-1", "app_version": "ics20-1"}
+    incentivized_args = (
+        [
+            "--channel-version",
+            json.dumps(version),
+        ]
+        if incentivized
+        else []
+    )
+
     subprocess.check_call(
         [
             "hermes",
@@ -44,10 +56,26 @@ def prepare_network(tmp_path, file):
             "--new-client-connection",
             "--yes",
         ]
+        + incentivized_args
     )
+
+    if incentivized:
+        # register fee payee
+        src_chain = cronos.cosmos_cli()
+        dst_chain = chainmain.cosmos_cli()
+        rsp = dst_chain.register_counterparty_payee(
+            "transfer",
+            "channel-0",
+            dst_chain.address("relayer"),
+            src_chain.address("signer1"),
+            from_="relayer",
+            fees="100000000basecro",
+        )
+        assert rsp["code"] == 0, rsp["raw_log"]
+
     supervisorctl(cronos.base_dir / "../tasks.ini", "start", "relayer-demo")
     wait_for_port(hermes.port)
-    yield IBCNetwork(cronos, chainmain, hermes)
+    yield IBCNetwork(cronos, chainmain, hermes, incentivized)
 
 
 def assert_ready(ibc):
@@ -58,7 +86,7 @@ def assert_ready(ibc):
     assert json.loads(output)["status"] == "success"
 
 
-def prepare(ibc):
+def hermes_transfer(ibc):
     assert_ready(ibc)
     # chainmain-1 -> cronos_777-1
     my_ibc0 = "chainmain-1"

integration_tests/shell.nix

@@ -17,8 +17,7 @@ pkgs.mkShell {
     pkgs.test-env
     pkgs.nixpkgs-fmt
     pkgs.rocksdb
-    (import ../nix/testenv.nix { inherit pkgs; })
-    (import ../nix/chainmain.nix { inherit pkgs; })
+    pkgs.chain-maind
     pkgs.hermes
   ];
 }