R4R: Fix state export/import, add to CI

.circleci/config.yml

@@ -137,6 +137,24 @@ jobs:
             export PATH="$GOBIN:$PATH"
             make test_sim_gaia_fast
 
+  test_sim_gaia_import_export:
+    <<: *defaults
+    parallelism: 1
+    steps:
+      - attach_workspace:
+          at: /tmp/workspace
+      - checkout
+      - run:
+          name: dependencies
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make get_vendor_deps
+      - run:
+          name: Test Gaia import/export simulation
+          command: |
+            export PATH="$GOBIN:$PATH"
+            make test_sim_gaia_import_export
+
   test_sim_gaia_multi_seed:
     <<: *defaults
     parallelism: 1
@@ -259,6 +277,9 @@ workflows:
       - test_sim_gaia_fast:
           requires:
             - setup_dependencies
+      - test_sim_gaia_import_export:
+          requires:
+            - setup_dependencies
       - test_sim_gaia_multi_seed:
           requires:
             - setup_dependencies

Makefile

@@ -171,6 +171,10 @@ test_sim_gaia_fast:
 	@echo "Running quick Gaia simulation. This may take several minutes..."
 	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=500 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=10 -v -timeout 24h
 
+test_sim_gaia_import_export:
+	@echo "Running Gaia import/export simulation. This may take several minutes..."
+	@go test ./cmd/gaia/app -run TestGaiaImportExport -SimulationEnabled=true -SimulationNumBlocks=50 -SimulationBlockSize=200 -SimulationCommit=true -SimulationSeed=12 -v -timeout 24h
+
 test_sim_gaia_multi_seed:
 	@echo "Running multi-seed Gaia simulation. This may take awhile!"
 	@bash scripts/multisim.sh 25

PENDING.md

@@ -28,7 +28,7 @@ FEATURES
 * Gaia
 
 * SDK
-    * (#1336) Mechanism for SDK Users to configure their own Bech32 prefixes instead of using the default cosmos prefixes.
+  - \#1336 Mechanism for SDK Users to configure their own Bech32 prefixes instead of using the default cosmos prefixes.
 
 * Tendermint
 
@@ -63,6 +63,7 @@ BUG FIXES
 
 * Gaia
  - \#2670 [x/stake] fixed incorrent `IterateBondedValidators` and split into two functions: `IterateBondedValidators` and `IterateLastBlockConsValidators`
+ - \#2648 [gaiad] Fix `gaiad export` / `gaiad import` consistency, test in CI
 
 * SDK
  - \#2625 [x/gov] fix AppendTag function usage error

cmd/gaia/app/app.go

@@ -231,6 +231,10 @@ func (app *GaiaApp) initChainer(ctx sdk.Context, req abci.RequestInitChain) abci
 		// return sdk.ErrGenesisParse("").TraceCause(err, "")
 	}
 
+	// sort by account number to maintain consistency
+	sort.Slice(genesisState.Accounts, func(i, j int) bool {
+		return genesisState.Accounts[i].AccountNumber < genesisState.Accounts[j].AccountNumber
+	})
 	// load the accounts
 	for _, gacc := range genesisState.Accounts {
 		acc := gacc.ToAccount()
@@ -245,6 +249,7 @@ func (app *GaiaApp) initChainer(ctx sdk.Context, req abci.RequestInitChain) abci
 	}
 
 	// load the address to pubkey map
+	auth.InitGenesis(ctx, app.feeCollectionKeeper, genesisState.AuthData)
 	slashing.InitGenesis(ctx, app.slashingKeeper, genesisState.SlashingData, genesisState.StakeData)
 	gov.InitGenesis(ctx, app.govKeeper, genesisState.GovData)
 	mint.InitGenesis(ctx, app.mintKeeper, genesisState.MintData)
@@ -306,11 +311,12 @@ func (app *GaiaApp) ExportAppStateAndValidators() (appState json.RawMessage, val
 	app.accountKeeper.IterateAccounts(ctx, appendAccount)
 	genState := NewGenesisState(
 		accounts,
+		auth.WriteGenesis(ctx, app.feeCollectionKeeper),
 		stake.WriteGenesis(ctx, app.stakeKeeper),
 		mint.WriteGenesis(ctx, app.mintKeeper),
 		distr.WriteGenesis(ctx, app.distrKeeper),
 		gov.WriteGenesis(ctx, app.govKeeper),
-		slashing.GenesisState{}, // TODO create write methods
+		slashing.WriteGenesis(ctx, app.slashingKeeper),
 	)
 	appState, err = codec.MarshalJSONIndent(app.cdc, genState)
 	if err != nil {

cmd/gaia/app/genesis.go

@@ -32,6 +32,7 @@ var (
 // State to Unmarshal
 type GenesisState struct {
 	Accounts     []GenesisAccount      `json:"accounts"`
+	AuthData     auth.GenesisState     `json:"auth"`
 	StakeData    stake.GenesisState    `json:"stake"`
 	MintData     mint.GenesisState     `json:"mint"`
 	DistrData    distr.GenesisState    `json:"distr"`
@@ -40,11 +41,12 @@ type GenesisState struct {
 	GenTxs       []json.RawMessage     `json:"gentxs"`
 }
 
-func NewGenesisState(accounts []GenesisAccount, stakeData stake.GenesisState, mintData mint.GenesisState,
+func NewGenesisState(accounts []GenesisAccount, authData auth.GenesisState, stakeData stake.GenesisState, mintData mint.GenesisState,
 	distrData distr.GenesisState, govData gov.GenesisState, slashingData slashing.GenesisState) GenesisState {
 
 	return GenesisState{
 		Accounts:     accounts,
+		AuthData:     authData,
 		StakeData:    stakeData,
 		MintData:     mintData,
 		DistrData:    distrData,
@@ -53,31 +55,39 @@ func NewGenesisState(accounts []GenesisAccount, stakeData stake.GenesisState, mi
 	}
 }
 
-// GenesisAccount doesn't need pubkey or sequence
+// nolint
 type GenesisAccount struct {
-	Address sdk.AccAddress `json:"address"`
-	Coins   sdk.Coins      `json:"coins"`
+	Address       sdk.AccAddress `json:"address"`
+	Coins         sdk.Coins      `json:"coins"`
+	Sequence      int64          `json:"sequence_number"`
+	AccountNumber int64          `json:"account_number"`
 }
 
 func NewGenesisAccount(acc *auth.BaseAccount) GenesisAccount {
 	return GenesisAccount{
-		Address: acc.Address,
-		Coins:   acc.Coins,
+		Address:       acc.Address,
+		Coins:         acc.Coins,
+		AccountNumber: acc.AccountNumber,
+		Sequence:      acc.Sequence,
 	}
 }
 
 func NewGenesisAccountI(acc auth.Account) GenesisAccount {
 	return GenesisAccount{
-		Address: acc.GetAddress(),
-		Coins:   acc.GetCoins(),
+		Address:       acc.GetAddress(),
+		Coins:         acc.GetCoins(),
+		AccountNumber: acc.GetAccountNumber(),
+		Sequence:      acc.GetSequence(),
 	}
 }
 
 // convert GenesisAccount to auth.BaseAccount
 func (ga *GenesisAccount) ToAccount() (acc *auth.BaseAccount) {
 	return &auth.BaseAccount{
-		Address: ga.Address,
-		Coins:   ga.Coins.Sort(),
+		Address:       ga.Address,
+		Coins:         ga.Coins.Sort(),
+		AccountNumber: ga.AccountNumber,
+		Sequence:      ga.Sequence,
 	}
 }
 

cmd/gaia/app/genesis_test.go

@@ -92,7 +92,7 @@ func makeMsg(name string, pk crypto.PubKey) auth.StdTx {
 	desc := stake.NewDescription(name, "", "", "")
 	comm := stakeTypes.CommissionMsg{}
 	msg := stake.NewMsgCreateValidator(sdk.ValAddress(pk.Address()), pk, sdk.NewInt64Coin(bondDenom,
-		50),	desc, comm)
+		50), desc, comm)
 	return auth.NewStdTx([]sdk.Msg{msg}, auth.StdFee{}, nil, "")
 }
 

cmd/gaia/app/sim_test.go

@@ -12,6 +12,7 @@ import (
 
 	"github.com/stretchr/testify/require"
 
+	abci "github.com/tendermint/tendermint/abci/types"
 	dbm "github.com/tendermint/tendermint/libs/db"
 	"github.com/tendermint/tendermint/libs/log"
 
@@ -266,6 +267,90 @@ func TestFullGaiaSimulation(t *testing.T) {
 	require.Nil(t, err)
 }
 
+func TestGaiaImportExport(t *testing.T) {
+	if !enabled {
+		t.Skip("Skipping Gaia import/export simulation")
+	}
+
+	// Setup Gaia application
+	var logger log.Logger
+	if verbose {
+		logger = log.TestingLogger()
+	} else {
+		logger = log.NewNopLogger()
+	}
+	var db dbm.DB
+	dir, _ := ioutil.TempDir("", "goleveldb-gaia-sim")
+	db, _ = dbm.NewGoLevelDB("Simulation", dir)
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+	app := NewGaiaApp(logger, db, nil)
+	require.Equal(t, "GaiaApp", app.Name())
+
+	// Run randomized simulation
+	err := simulation.SimulateFromSeed(
+		t, app.BaseApp, appStateFn, seed,
+		testAndRunTxs(app),
+		[]simulation.RandSetup{},
+		invariants(app),
+		numBlocks,
+		blockSize,
+		commit,
+	)
+	if commit {
+		// for memdb:
+		// fmt.Println("Database Size", db.Stats()["database.size"])
+		fmt.Println("GoLevelDB Stats")
+		fmt.Println(db.Stats()["leveldb.stats"])
+		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
+	}
+	require.Nil(t, err)
+
+	fmt.Printf("Exporting genesis...\n")
+
+	appState, _, err := app.ExportAppStateAndValidators()
+	if err != nil {
+		panic(err)
+	}
+
+	fmt.Printf("Importing genesis...\n")
+
+	newDir, _ := ioutil.TempDir("", "goleveldb-gaia-sim-2")
+	newDB, _ := dbm.NewGoLevelDB("Simulation-2", dir)
+	defer func() {
+		newDB.Close()
+		os.RemoveAll(newDir)
+	}()
+	newApp := NewGaiaApp(log.NewNopLogger(), newDB, nil)
+	require.Equal(t, "GaiaApp", newApp.Name())
+	request := abci.RequestInitChain{
+		AppStateBytes: appState,
+	}
+	newApp.InitChain(request)
+	newApp.Commit()
+
+	fmt.Printf("Comparing stores...\n")
+	ctxA := app.NewContext(true, abci.Header{})
+	ctxB := newApp.NewContext(true, abci.Header{})
+	storeKeysA := []sdk.StoreKey{app.keyMain, app.keyAccount, app.keyStake, app.keySlashing,
+		app.keyMint, app.keyDistr, app.keyFeeCollection, app.keyParams, app.keyGov}
+	storeKeysB := []sdk.StoreKey{newApp.keyMain, newApp.keyAccount, newApp.keyStake, newApp.keySlashing,
+		newApp.keyMint, newApp.keyDistr, newApp.keyFeeCollection, newApp.keyParams, newApp.keyGov}
+	require.Equal(t, len(storeKeysA), len(storeKeysB))
+	for index, storeKeyA := range storeKeysA {
+		storeKeyB := storeKeysB[index]
+		storeA := ctxA.KVStore(storeKeyA)
+		storeB := ctxB.KVStore(storeKeyB)
+		kvA, kvB, count, equal := sdk.DiffKVStores(storeA, storeB)
+		fmt.Printf("Compared %d key/value pairs between %s and %s\n", count, storeKeyA, storeKeyB)
+		require.True(t, equal, "unequal stores: %s / %s:\nstore A %s (%X) => %s (%X)\nstore B %s (%X) => %s (%X)",
+			storeKeyA, storeKeyB, kvA.Key, kvA.Key, kvA.Value, kvA.Value, kvB.Key, kvB.Key, kvB.Value, kvB.Value)
+	}
+
+}
+
 // TODO: Make another test for the fuzzer itself, which just has noOp txs
 // and doesn't depend on gaia
 func TestAppStateDeterminism(t *testing.T) {

types/store.go

@@ -1,6 +1,7 @@
 package types
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 
@@ -176,6 +177,33 @@ func KVStoreReversePrefixIterator(kvs KVStore, prefix []byte) Iterator {
 	return kvs.ReverseIterator(prefix, PrefixEndBytes(prefix))
 }
 
+// Compare two KVstores, return either the first key/value pair
+// at which they differ and whether or not they are equal
+func DiffKVStores(a KVStore, b KVStore) (kvA cmn.KVPair, kvB cmn.KVPair, count int64, equal bool) {
+	iterA := a.Iterator(nil, nil)
+	iterB := b.Iterator(nil, nil)
+	count = int64(0)
+	for {
+		if !iterA.Valid() && !iterB.Valid() {
+			break
+		}
+		var kvA, kvB cmn.KVPair
+		if iterA.Valid() {
+			kvA = cmn.KVPair{Key: iterA.Key(), Value: iterA.Value()}
+		}
+		if iterB.Valid() {
+			kvB = cmn.KVPair{Key: iterB.Key(), Value: iterB.Value()}
+		}
+		if !bytes.Equal(kvA.Key, kvB.Key) || !bytes.Equal(kvA.Value, kvB.Value) {
+			return kvA, kvB, count, false
+		}
+		iterA.Next()
+		iterB.Next()
+		count++
+	}
+	return cmn.KVPair{}, cmn.KVPair{}, count, true
+}
+
 // CacheKVStore cache-wraps a KVStore.  After calling .Write() on
 // the CacheKVStore, all previously created CacheKVStores on the
 // object expire.

x/auth/genesis.go

@@ -0,0 +1,33 @@
+package auth
+
+import (
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+// GenesisState - all auth state that must be provided at genesis
+type GenesisState struct {
+	CollectedFees sdk.Coins `json:"collected_fees"` // collected fees
+}
+
+// Create a new genesis state
+func NewGenesisState(collectedFees sdk.Coins) GenesisState {
+	return GenesisState{
+		CollectedFees: collectedFees,
+	}
+}
+
+// Return a default genesis state
+func DefaultGenesisState() GenesisState {
+	return NewGenesisState(sdk.Coins{})
+}
+
+// Init store state from genesis data
+func InitGenesis(ctx sdk.Context, keeper FeeCollectionKeeper, data GenesisState) {
+	keeper.setCollectedFees(ctx, data.CollectedFees)
+}
+
+// WriteGenesis returns a GenesisState for a given context and keeper
+func WriteGenesis(ctx sdk.Context, keeper FeeCollectionKeeper) GenesisState {
+	collectedFees := keeper.GetCollectedFees(ctx)
+	return NewGenesisState(collectedFees)
+}

x/distribution/genesis.go

@@ -21,6 +21,7 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data types.GenesisState) {
 	for _, dw := range data.DelegatorWithdrawInfos {
 		keeper.SetDelegatorWithdrawAddr(ctx, dw.DelegatorAddr, dw.WithdrawAddr)
 	}
+	keeper.SetPreviousProposerConsAddr(ctx, data.PreviousProposer)
 }
 
 // WriteGenesis returns a GenesisState for a given context and keeper. The
@@ -33,6 +34,7 @@ func WriteGenesis(ctx sdk.Context, keeper Keeper) types.GenesisState {
 	vdis := keeper.GetAllValidatorDistInfos(ctx)
 	ddis := keeper.GetAllDelegationDistInfos(ctx)
 	dwis := keeper.GetAllDelegatorWithdrawInfos(ctx)
+	pp := keeper.GetPreviousProposerConsAddr(ctx)
 	return NewGenesisState(feePool, communityTax, baseProposerRewards,
-		bonusProposerRewards, vdis, ddis, dwis)
+		bonusProposerRewards, vdis, ddis, dwis, pp)
 }

x/distribution/keeper/genesis.go

@@ -36,12 +36,12 @@ func (k Keeper) GetAllDelegationDistInfos(ctx sdk.Context) (ddis []types.Delegat
 // Get the set of all delegator-withdraw addresses with no limits, used during genesis dump
 func (k Keeper) GetAllDelegatorWithdrawInfos(ctx sdk.Context) (dwis []types.DelegatorWithdrawInfo) {
 	store := ctx.KVStore(k.storeKey)
-	iterator := sdk.KVStorePrefixIterator(store, DelegationDistInfoKey)
+	iterator := sdk.KVStorePrefixIterator(store, DelegatorWithdrawInfoKey)
 	defer iterator.Close()
 
 	for ; iterator.Valid(); iterator.Next() {
 		dw := types.DelegatorWithdrawInfo{
-			DelegatorAddr: sdk.AccAddress(iterator.Key()),
+			DelegatorAddr: sdk.AccAddress(iterator.Key()[1:]),
 			WithdrawAddr:  sdk.AccAddress(iterator.Value()),
 		}
 		dwis = append(dwis, dw)