Standalone to consumer changeover - recycle existing transfer channel (…

…#832)

* on-chain upgrade to consumer chain wip

* add preCCV store and use it on democracy staking

* add TODOs and one more packet possibility

* status update

* Resolve hermes start issue for trusted validator set by changing revision height

* remove intermediary logs

* remove further unused codebase

* updates for endblocker test, existing test fixes, get last validators

* update for slashing sovereign validators for the fault made before consumer chain upgrade height

* resolve comments on github and slack communication

* update sovereign app to use v4 ibc from v3 & resolve consumer module merge conflict fix issue

* Update app/sovereign/upgrades/v3/upgrades.go

Co-authored-by: yaruwangway <69694322+yaruwangway@users.noreply.github.com>

* rm sovereign chain and tests. Will be replaced by simapp and integration tests

* duplicate module name

* add comment

* small rename

* remove democracy staking changes

* consumer ccv beginblock, endblock, and initgenesis order shouldn't matter

* add mock calls to compile

* adjust tests for new keeper field

* add registerDemocConsumer method

* split out preCCV flag and initial valset

* cleanup consumer module

* cleanup

* more cleanup

* temp changes to validators.go

* comment out test

* rm bad code from merge

* comment

* Update app.go

* UTs for CRUD

* UTs for keys

* use make for mocks

* todo

* changeover method and test

* resolve #783

* comment

* comments

* add appropriate TODOs, restore changes to main

* final nits before non-draft

* comment on ChangeoverToConsumer

* more clear comment

* small comment change

* update InitGenesis comment

* sovereign -> standalone

* missed a file

* builds now

* update comment after debug

* ChangeoverIsComplete method

* small refactors, comments, slash and jail code

* naming refactor

* fix last standalone height

* Update module.go

* needed refactors

* rm unneeded diffs

* two UTs

* sanity check integration test

* gofumpt

* last standalone height -> init genesis height

* wiring

* integration test and refactor to enable it

* sanity check

* Update changeover.go

* Use ValidatorUpdateDelay

* for loop

* fix key name

* Update x/ccv/consumer/keeper/changeover.go

Co-authored-by: Simon Noetzlin <simon.ntz@gmail.com>

* Update x/ccv/consumer/keeper/validators.go

Co-authored-by: Simon Noetzlin <simon.ntz@gmail.com>

* fix test

* rm dup method

* Update keys_test.go

---------

Co-authored-by: jstr1121 <jun@stridelabs.co>
Co-authored-by: jstr1121 <118450565+jstr1121@users.noreply.github.com>
Co-authored-by: yaruwangway <69694322+yaruwangway@users.noreply.github.com>
Co-authored-by: Simon Noetzlin <simon.ntz@gmail.com>

app/consumer-democracy/app.go

@@ -643,6 +643,12 @@ func New(
 			// upgrade handler code is application specific. However, as an example, standalone to consumer
 			// changeover chains should utilize customized upgrade handler code similar to below.
 
+			// Setting the standalone transfer channel ID is only needed for standalone to consumer changeover chains
+			// who wish to preserve existing IBC transfer denoms. Here's an example.
+			//
+			// Note: This setter needs to execute before the ccv channel handshake is initiated.
+			app.ConsumerKeeper.SetStandaloneTransferChannelID(ctx, "hardcoded-existing-channel-id")
+
 			// TODO: should have a way to read from current node home
 			userHomeDir, err := os.UserHomeDir()
 			if err != nil {

tests/integration/changeover.go

@@ -0,0 +1,53 @@
+package integration
+
+import (
+	transfertypes "github.com/cosmos/ibc-go/v4/modules/apps/transfer/types"
+	channeltypes "github.com/cosmos/ibc-go/v4/modules/core/04-channel/types"
+)
+
+func (suite *CCVTestSuite) TestRecycleTransferChannel() {
+	consumerKeeper := suite.consumerApp.GetConsumerKeeper()
+
+	// Only create a connection between consumer and provider
+	suite.coordinator.CreateConnections(suite.path)
+
+	// Confirm transfer channel has not been persisted
+	transChan := consumerKeeper.GetDistributionTransmissionChannel(suite.consumerCtx())
+	suite.Require().Empty(transChan)
+
+	// Create transfer channel manually
+	distrTransferMsg := channeltypes.NewMsgChannelOpenInit(
+		transfertypes.PortID,
+		transfertypes.Version,
+		channeltypes.UNORDERED,
+		[]string{suite.path.EndpointA.ConnectionID},
+		transfertypes.PortID,
+		"", // signer unused
+	)
+	resp, err := consumerKeeper.ChannelOpenInit(suite.consumerCtx(), distrTransferMsg)
+	suite.Require().NoError(err)
+
+	// Confirm transfer channel still not persisted
+	transChan = consumerKeeper.GetDistributionTransmissionChannel(suite.consumerCtx())
+	suite.Require().Empty(transChan)
+
+	// Setup state s.t. the consumer keeper emulates a consumer that was previously standalone
+	consumerKeeper.MarkAsPrevStandaloneChain(suite.consumerCtx())
+	suite.Require().True(consumerKeeper.IsPrevStandaloneChain(suite.consumerCtx()))
+	suite.consumerApp.GetConsumerKeeper().SetStandaloneTransferChannelID(suite.consumerCtx(), resp.ChannelId)
+
+	// Now finish setting up CCV channel
+	suite.ExecuteCCVChannelHandshake(suite.path)
+
+	// Confirm transfer channel is now persisted with expected channel id from open init response
+	transChan = consumerKeeper.GetDistributionTransmissionChannel(suite.consumerCtx())
+	suite.Require().Equal(resp.ChannelId, transChan)
+
+	// Confirm channel exists
+	found := consumerKeeper.TransferChannelExists(suite.consumerCtx(), transChan)
+	suite.Require().True(found)
+
+	// Sanity check, only two channels should exist, one transfer and one ccv
+	channels := suite.consumerApp.GetIBCKeeper().ChannelKeeper.GetAllChannels(suite.consumerCtx())
+	suite.Require().Len(channels, 2)
+}

tests/integration/setup.go

@@ -239,7 +239,10 @@ func (suite *CCVTestSuite) SetupAllCCVChannels() {
 
 func (suite *CCVTestSuite) SetupCCVChannel(path *ibctesting.Path) {
 	suite.coordinator.CreateConnections(path)
+	suite.ExecuteCCVChannelHandshake(path)
+}
 
+func (suite *CCVTestSuite) ExecuteCCVChannelHandshake(path *ibctesting.Path) {
 	err := path.EndpointA.ChanOpenInit()
 	suite.Require().NoError(err)
 

testutil/integration/debug_test.go

@@ -244,3 +244,11 @@ func TestPacketRoundtrip(t *testing.T) {
 func TestQueueAndSendVSCMaturedPackets(t *testing.T) {
 	runCCVTestByName(t, "TestQueueAndSendVSCMaturedPackets")
 }
+
+//
+// Changeover tests
+//
+
+func TestRecycleTransferChannel(t *testing.T) {
+	runCCVTestByName(t, "TestRecycleTransferChannel")
+}

x/ccv/consumer/ibc_module.go

@@ -138,7 +138,18 @@ func (am AppModule) OnChanOpenAck(
 
 	///////////////////////////////////////////////////
 	// Initialize distribution token transfer channel
-	//
+
+	// First check if an existing transfer channel exists, if this consumer was a previously standalone chain.
+	if am.keeper.IsPrevStandaloneChain(ctx) {
+		transChannelID := am.keeper.GetStandaloneTransferChannelID(ctx)
+		found := am.keeper.TransferChannelExists(ctx, transChannelID)
+		if found {
+			// If existing transfer channel is found, persist that channel ID and return
+			am.keeper.SetDistributionTransmissionChannel(ctx, transChannelID)
+			return nil
+		}
+	}
+
 	// NOTE The handshake for this channel is handled by the ibc-go/transfer
 	// module. If the transfer-channel fails here (unlikely) then the transfer
 	// channel should be manually created and ccv parameters set accordingly.

x/ccv/consumer/keeper/changeover.go

@@ -7,7 +7,7 @@ import (
 
 // ChangeoverIsComplete returns whether the standalone to consumer changeover process is complete.
 func (k Keeper) ChangeoverIsComplete(ctx sdk.Context) bool {
-	if !k.IsPrevStandaloneChain() {
+	if !k.IsPrevStandaloneChain(ctx) {
 		panic("ChangeoverIsComplete should only be called on previously standalone consumers")
 	}
 	return ctx.BlockHeight() >= k.FirstConsumerHeight(ctx)

x/ccv/consumer/keeper/distribution.go

@@ -178,6 +178,11 @@ func (k Keeper) ChannelOpenInit(ctx sdk.Context, msg *channeltypes.MsgChannelOpe
 	return k.ibcCoreKeeper.ChannelOpenInit(sdk.WrapSDKContext(ctx), msg)
 }
 
+func (k Keeper) TransferChannelExists(ctx sdk.Context, channelID string) bool {
+	_, found := k.channelKeeper.GetChannel(ctx, transfertypes.PortID, channelID)
+	return found
+}
+
 func (k Keeper) GetConnectionHops(ctx sdk.Context, srcPort, srcChan string) ([]string, error) {
 	ch, found := k.channelKeeper.GetChannel(ctx, srcPort, srcChan)
 	if !found {

x/ccv/consumer/keeper/genesis.go

@@ -25,6 +25,7 @@ func (k Keeper) InitGenesis(ctx sdk.Context, state *consumertypes.GenesisState)
 	// stick around for slashing/jailing purposes.
 	if state.PreCCV {
 		k.SetPreCCVTrue(ctx)
+		k.MarkAsPrevStandaloneChain(ctx)
 		k.SetInitialValSet(ctx, state.InitialValSet)
 	}
 	k.SetInitGenesisHeight(ctx, ctx.BlockHeight()) // Usually 0, but not the case for changeover chains

x/ccv/consumer/keeper/keeper.go

@@ -92,10 +92,6 @@ func (k *Keeper) SetStandaloneStakingKeeper(sk ccv.StakingKeeper) {
 	k.standaloneStakingKeeper = sk
 }
 
-func (k Keeper) IsPrevStandaloneChain() bool {
-	return k.standaloneStakingKeeper != nil
-}
-
 // Validates that the consumer keeper is initialized with non-zero and
 // non-nil values for all its fields. Otherwise this method will panic.
 func (k Keeper) mustValidateFields() {
@@ -261,7 +257,7 @@ func (k Keeper) DeletePendingChanges(ctx sdk.Context) {
 
 func (k Keeper) GetInitGenesisHeight(ctx sdk.Context) int64 {
 	store := ctx.KVStore(k.storeKey)
-	bz := store.Get(types.LastStandaloneHeightKey())
+	bz := store.Get(types.InitGenesisHeightKey())
 	if bz == nil {
 		panic("last standalone height not set")
 	}
@@ -272,7 +268,7 @@ func (k Keeper) GetInitGenesisHeight(ctx sdk.Context) int64 {
 func (k Keeper) SetInitGenesisHeight(ctx sdk.Context, height int64) {
 	bz := sdk.Uint64ToBigEndian(uint64(height))
 	store := ctx.KVStore(k.storeKey)
-	store.Set(types.LastStandaloneHeightKey(), bz)
+	store.Set(types.InitGenesisHeightKey(), bz)
 }
 
 func (k Keeper) IsPreCCV(ctx sdk.Context) bool {
@@ -606,3 +602,27 @@ func (k Keeper) AppendPendingPacket(ctx sdk.Context, packet ...ccv.ConsumerPacke
 	list := append(pending.GetList(), packet...)
 	k.SetPendingPackets(ctx, ccv.ConsumerPacketDataList{List: list})
 }
+
+func (k Keeper) MarkAsPrevStandaloneChain(ctx sdk.Context) {
+	store := ctx.KVStore(k.storeKey)
+	store.Set(types.PrevStandaloneChainKey(), []byte{})
+}
+
+func (k Keeper) IsPrevStandaloneChain(ctx sdk.Context) bool {
+	store := ctx.KVStore(k.storeKey)
+	return store.Has(types.PrevStandaloneChainKey())
+}
+
+// SetStandaloneTransferChannelID sets the channelID of an existing transfer channel,
+// for a chain which used to be a standalone chain.
+func (k Keeper) SetStandaloneTransferChannelID(ctx sdk.Context, channelID string) {
+	store := ctx.KVStore(k.storeKey)
+	store.Set(types.StandaloneTransferChannelIDKey(), []byte(channelID))
+}
+
+// GetStandaloneTransferChannelID returns the channelID of an existing transfer channel,
+// for a chain which used to be a standalone chain.
+func (k Keeper) GetStandaloneTransferChannelID(ctx sdk.Context) string {
+	store := ctx.KVStore(k.storeKey)
+	return string(store.Get(types.StandaloneTransferChannelIDKey()))
+}

x/ccv/consumer/keeper/keeper_test.go

@@ -540,3 +540,28 @@ func TestGetAllOutstandingDowntimes(t *testing.T) {
 	require.Len(t, result, len(addresses))
 	require.Equal(t, result, expectedGetAllOrder)
 }
+
+// TestStandaloneTransferChannelID tests the getter and setter for the existing transfer channel id
+func TestStandaloneTransferChannelID(t *testing.T) {
+	ck, ctx, ctrl, _ := testkeeper.GetConsumerKeeperAndCtx(t, testkeeper.NewInMemKeeperParams(t))
+	defer ctrl.Finish()
+
+	// Test that the default value is empty
+	require.Equal(t, "", ck.GetStandaloneTransferChannelID(ctx))
+
+	// Test that the value can be set and retrieved
+	ck.SetStandaloneTransferChannelID(ctx, "channelID1234")
+	require.Equal(t, "channelID1234", ck.GetStandaloneTransferChannelID(ctx))
+}
+
+func TestPrevStandaloneChainFlag(t *testing.T) {
+	ck, ctx, ctrl, _ := testkeeper.GetConsumerKeeperAndCtx(t, testkeeper.NewInMemKeeperParams(t))
+	defer ctrl.Finish()
+
+	// Test that the default value is false
+	require.False(t, ck.IsPrevStandaloneChain(ctx))
+
+	// Test that the value can be set and retrieved
+	ck.MarkAsPrevStandaloneChain(ctx)
+	require.True(t, ck.IsPrevStandaloneChain(ctx))
+}

x/ccv/consumer/keeper/validators.go

@@ -80,7 +80,7 @@ func (k Keeper) Validator(ctx sdk.Context, addr sdk.ValAddress) stakingtypes.Val
 func (k Keeper) IsValidatorJailed(ctx sdk.Context, addr sdk.ConsAddress) bool {
 	// if the changeover is not complete for prev standalone chain,
 	// return the standalone staking keeper's jailed status
-	if k.IsPrevStandaloneChain() && !k.ChangeoverIsComplete(ctx) {
+	if k.IsPrevStandaloneChain(ctx) && !k.ChangeoverIsComplete(ctx) {
 		return k.standaloneStakingKeeper.IsValidatorJailed(ctx, addr)
 	}
 	// Otherwise, return the ccv consumer keeper's notion of a validator being jailed
@@ -111,7 +111,7 @@ func (k Keeper) Slash(ctx sdk.Context, addr sdk.ConsAddress, infractionHeight, p
 
 	// If this is a previously standalone chain and infraction happened before the changeover was completed,
 	// slash only on the standalone staking keeper.
-	if k.IsPrevStandaloneChain() && infractionHeight < k.FirstConsumerHeight(ctx) {
+	if k.IsPrevStandaloneChain(ctx) && infractionHeight < k.FirstConsumerHeight(ctx) {
 		k.standaloneStakingKeeper.Slash(ctx, addr, infractionHeight, power, slashFactor, stakingtypes.InfractionEmpty)
 		return
 	}

x/ccv/consumer/keeper/validators_test.go

@@ -111,7 +111,7 @@ func TestIsValidatorJailed(t *testing.T) {
 	defer ctrl.Finish()
 
 	// Consumer keeper from test setup should return false for IsPrevStandaloneChain()
-	require.False(t, consumerKeeper.IsPrevStandaloneChain())
+	require.False(t, consumerKeeper.IsPrevStandaloneChain(ctx))
 
 	// IsValidatorJailed should return false for an arbitrary consensus address
 	consAddr := []byte{0x01, 0x02, 0x03}
@@ -123,9 +123,11 @@ func TestIsValidatorJailed(t *testing.T) {
 	// Now confirm IsValidatorJailed returns true
 	require.True(t, consumerKeeper.IsValidatorJailed(ctx, consAddr))
 
-	// Next, we set a value for the standalone staking keeper so IsPrevStandaloneChain() returns true
+	// Next, we set a value for the standalone staking keeper,
+	// and mark the consumer keeper as being from a previous standalone chain
 	consumerKeeper.SetStandaloneStakingKeeper(mocks.MockStakingKeeper)
-	require.True(t, consumerKeeper.IsPrevStandaloneChain())
+	consumerKeeper.MarkAsPrevStandaloneChain(ctx)
+	require.True(t, consumerKeeper.IsPrevStandaloneChain(ctx))
 
 	// Set init genesis height to current block height so that ChangeoverIsComplete() is false
 	consumerKeeper.SetInitGenesisHeight(ctx, ctx.BlockHeight())
@@ -150,7 +152,7 @@ func TestSlash(t *testing.T) {
 	require.Len(t, pendingPackets.List, 0)
 
 	// Consumer keeper from test setup should return false for IsPrevStandaloneChain()
-	require.False(t, consumerKeeper.IsPrevStandaloneChain())
+	require.False(t, consumerKeeper.IsPrevStandaloneChain(ctx))
 
 	// Now setup a value for vscID mapped to infraction height
 	consumerKeeper.SetHeightValsetUpdateID(ctx, 5, 6)
@@ -160,9 +162,11 @@ func TestSlash(t *testing.T) {
 	pendingPackets = consumerKeeper.GetPendingPackets(ctx)
 	require.Len(t, pendingPackets.List, 1)
 
-	// Next, we set a value for the standalone staking keeper so IsPrevStandaloneChain() returns true
+	// Next, we set a value for the standalone staking keeper,
+	// and mark the consumer keeper as being from a previous standalone chain
 	consumerKeeper.SetStandaloneStakingKeeper(mocks.MockStakingKeeper)
-	require.True(t, consumerKeeper.IsPrevStandaloneChain())
+	consumerKeeper.MarkAsPrevStandaloneChain(ctx)
+	require.True(t, consumerKeeper.IsPrevStandaloneChain(ctx))
 
 	// At this point, the state of the consumer keeper is s.t.
 	// Slash() calls the standalone staking keeper's Slash()

x/ccv/consumer/types/keys.go

@@ -61,12 +61,19 @@ const (
 	// InitialValSetByteKey is the byte to store the initial validator set for a consumer
 	InitialValSetByteKey
 
-	// LastStandaloneHeightByteKey is the byte that will store last standalone height
-	LastStandaloneHeightByteKey
+	// InitGenesisHeightByteKey is the byte that will store the init genesis height
+	InitGenesisHeightByteKey
 
 	// SmallestNonOptOutPowerByteKey is the byte that will store the smallest val power that cannot opt out
 	SmallestNonOptOutPowerByteKey
 
+	// StandaloneTransferChannelIDByteKey is the byte storing the channelID of transfer channel
+	// that existed from a standalone chain changing over to a consumer
+	StandaloneTransferChannelIDByteKey
+
+	// PrevStandaloneChainByteKey is the byte storing the flag marking whether this chain was previously standalone
+	PrevStandaloneChainByteKey
+
 	// HistoricalInfoKey is the byte prefix that will store the historical info for a given height
 	HistoricalInfoBytePrefix
 
@@ -134,14 +141,25 @@ func InitialValSetKey() []byte {
 	return []byte{InitialValSetByteKey}
 }
 
-func LastStandaloneHeightKey() []byte {
-	return []byte{LastStandaloneHeightByteKey}
+func InitGenesisHeightKey() []byte {
+	return []byte{InitGenesisHeightByteKey}
 }
 
 func SmallestNonOptOutPowerKey() []byte {
 	return []byte{SmallestNonOptOutPowerByteKey}
 }
 
+// StandaloneTransferChannelIDKey returns the key to the transfer channelID that existed from a standalone chain
+// changing over to a consumer
+func StandaloneTransferChannelIDKey() []byte {
+	return []byte{StandaloneTransferChannelIDByteKey}
+}
+
+// PrevStandaloneChainKey returns the key to the flag marking whether this chain was previously standalone
+func PrevStandaloneChainKey() []byte {
+	return []byte{PrevStandaloneChainByteKey}
+}
+
 // HistoricalInfoKey returns the key to historical info to a given block height
 func HistoricalInfoKey(height int64) []byte {
 	hBytes := make([]byte, 8)

x/ccv/consumer/types/keys_test.go

@@ -30,8 +30,10 @@ func getAllKeyPrefixes() []byte {
 		PendingDataPacketsByteKey,
 		PreCCVByteKey,
 		InitialValSetByteKey,
-		LastStandaloneHeightByteKey,
+		InitGenesisHeightByteKey,
 		SmallestNonOptOutPowerByteKey,
+		StandaloneTransferChannelIDByteKey,
+		PrevStandaloneChainByteKey,
 		HistoricalInfoBytePrefix,
 		PacketMaturityTimeBytePrefix,
 		HeightValsetUpdateIDBytePrefix,
@@ -62,8 +64,10 @@ func getAllFullyDefinedKeys() [][]byte {
 		PendingDataPacketsKey(),
 		PreCCVKey(),
 		InitialValSetKey(),
-		LastStandaloneHeightKey(),
+		InitGenesisHeightKey(),
 		SmallestNonOptOutPowerKey(),
+		StandaloneTransferChannelIDKey(),
+		PrevStandaloneChainKey(),
 		HistoricalInfoKey(0),
 		PacketMaturityTimeKey(0, time.Time{}),
 		HeightValsetUpdateIDKey(0),