feat(migrations): Added migration tests basic frame & PoC (#418)

* Debugging proposal versioning * Fixed to legacy types * Fixed query return type * Added impl check * Added extended registry * `gofumpt -ed` * Added current def * Added shorter voting period by default * Fixed to a tad shorter voting period * Added clean env image load * Fixed paths & permissions * `gofumpt -ed` * Bump `ginkgo@latest` & improvements * unstash changes * Merge with protobuf changes * Cleaning up * Debugging reboot with new image * Debugging * Debugging * Added baseapp module migration order * Removed buf breaking wf temporarily * Skipped capability from version map * Skipped distribution module directly to v2 * Skipped staking module directly to v3 * Added `min_commission_rates` to genesis & improvements * Fixed traverse * Added switch to fresh genesis * Switched to pre-halt * Added subspace persist * Removed interactive * Upgrade works, debugging status ping * Added larger time window for catching up after upgrade * Fixed time window * Added much larger time window * Some changes for syncing * Fixed multiline prepended cli output * Fixed declaration * Add protobuf migration * Backwards compatibility for gas estimate print * Bump digits EOF inclusive * Switched to caught up validator * Added deep equality extra cases * Fixed key mismatch * Fixed assertion * Add some changes for resource migrations * Added distinct steps prints * Cleanup * Update build.yml * Create mainnet-latest.env * Rename build file * Update docker-compose.yml * small renamings * Additional broken changes to workflow * Checkpoint * Refactor for Proto v2 * Removed unused definition * Removed extra * Added pre upgrade step & compose improvements * Fixed carry * Removed bloat * Removed bloat * Added external pull * Debugging on latest * Debugging * Added latest binary for config generation * Fixed dir * Debugging * Removed operator key defs * Removed trips * Added update, deactivate , resource cases & improvements * Added temporary key copies * Restructured with temporary steps * Added image load * Fixed step * Fixed cwd * Fixed cwd * Added env * Restored permissions * Switched to full dir * Switched to multibase char rather than bytes & improvements * Fixed resource file mismatch * Fixed carry * Added alternate base58 payload * Added assertion method * Added post & improvements on pre * Triggering fresh run * Debugging * Added temp wait * Fixed versioning * Added status ping on catching up & improvements * Fixed versioning * Fixed literals & improvements on versioning * Fixed literals * Fixed versioning * Switched to legacy query * Added module version map match & v1 proto keeper, msg server * Added distinction * Fixed deps * Synced across migration handlers & improvements on scenario handling * Added nolint rules to scaffolds * Removed extra genesis record, will be needed to start afresh * Removed deprecated part * Switched to login * Trying out bind volumes as operator keys * Debugging * Removed bound volumes, since nodes are not syncing * Finish Indy-style migration * Make it buildable after merging with develop * Added unified payload - sign input logic * Temp takedown of buf breaking changes linter * Fixed error handling & improvements * Temporarily disable unit upgrade tests * Debugging * Added new temporary test setup * Run the first scenario * Move upgrade unit tests into a separate stage * Fix linter issues * Add setup and cleanup scripts * Exptract steps into bash scripts * Fix permissions * Premissions fix * refactor test migrator and add tests for checksum and protobuf * Makes linter happy * Cleanup * Fix build * Debugging * Add JUnit report for Upgrade step * Make protobuf and checksum tests work * Separate unit and integration upgrade test * Upgrade test suits * Fix namespaces * Minor fixes * Extract proto related migrations into it's own files * Commit for sync * Fix merge issues * Fixes * Commit for sync * Sync * Misc * Make IndyStyle migration works * Merge with latest changes * Makes linter happy * Reviewing migrations * Fix bug * Fix merge conflicts * Fix iterator issue * Run test pipelines * Clean up * Review checksumm migration * Restore build tags * Extract simple migration helpers * Implement version id migration * Unstash * Run it on pipelines * Fix tests running on pipelines * Fix cleanup * Fix test reporting * Remove redundant files * Add generic dataset implementation * Cleanup * Cleanup * Add uuid migration * Add resource version links migration * Fix build * Add unit tests for UUID (#456) * Move version setting to SetResource method * Make UUID tests for migration * Add tests for VersionId and create AddNewVersionMethod * Clenup Co-authored-by: Alexandr Kolesov <kolesov.alex.s@gmail.com> * Fix version links migration * Restore version links migration * Cleanup * Vresion map fix attempt * Cleanup * Simplify unit test workflow * Add more migration handlers * Update test.yml * Try disabling version map fix * Uncomment integration tests * Reorganize file structure * Restore vm fix, fix linter * Add resource links test (#457) Add tests for resource links * Add uuid migration for resource id * Commit for sync * add checksum to datasets * Add diddoc pre update * Fix migration unit tests * Merge with latest version * Fix tests for post checks * Makes linter happy * Fix pipelines * Fix resource unit tests * Better test structure * Fix loader * Debugging post did update * For the previous commit * Fix signature verification bug * Add did deactionvation * Fix update and deactivate cases * DIDDoc queries update * Update tx_create_diddoc.go * Update tx_deactivate_diddoc.go * Update tx_update_diddoc.go * Update query_diddoc.go * Rename create-resource * Update go.mod * Update migration_did_protobuf.go * Add resource default alternative url migration * Enable default alternative url migration + add test for it, fix build * Added resource creation after upgrade * Fix tests * Rename create-resource to create for tests * Move resource creation before queries * Fix resource tests * Move directory structure * Fix tests after directory migration * Temporary disable markdown linting * Remove unused constants * Implement all versions deactivation * Fix linter issues * Fix broken link Co-authored-by: Andrew Nikitin <andrew.nikitin@cheqd.io> Co-authored-by: Alexandr Kolesov <kolesov.alex.s@gmail.com> Co-authored-by: Ankur Banerjee <ankurdotb@users.noreply.github.com>
cheqd · Dec 7, 2022 · bf1f48b · bf1f48b
1 parent 3f541ec
commit bf1f48b
Show file tree

Hide file tree

Showing 181 changed files with 7,517 additions and 81 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -64,10 +64,10 @@ jobs:
       - uses: bufbuild/buf-lint-action@v1
 
       # Breaking change detection
-      # - uses: bufbuild/buf-breaking-action@v1
-      #   with:
-      #     input: 'proto'
-      #     against: 'https://github.com/cheqd/cheqd-node.git#branch=develop,ref=HEAD~1,subdir=proto'
+      - uses: bufbuild/buf-breaking-action@v1
+        with:
+          input: 'proto'
+          against: 'https://github.com/cheqd/cheqd-node.git#branch=develop,ref=HEAD~1,subdir=proto'
 
   super-lint:
     name: "Super Linter"

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -30,8 +30,8 @@ jobs:
         run: |
           go install github.com/onsi/ginkgo/v2/ginkgo@latest
 
-      - name: Run Ginkgo tests
-        run: ginkgo -r --race --randomize-all --randomize-suites --keep-going --trace --junit-report report-unit.xml
+      - name: Run Ginkgo unit tests
+        run: ginkgo -r --race --randomize-all --randomize-suites --keep-going --trace --tags upgrade_unit --junit-report report-unit.xml
 
       - name: Upload unit tests result
         uses: actions/upload-artifact@v3
@@ -40,7 +40,7 @@ jobs:
           path: report-unit.xml
 
   integration-tests:
-    name: "Integration Test"
+    name: "Integration Tests"
     runs-on: ubuntu-latest
 
     steps:
@@ -73,7 +73,7 @@ jobs:
       - name: Set up Docker localnet
         working-directory: ./docker/localnet
         run: |
-          docker compose --env-file docker-compose.env up --detach --no-build
+          docker compose --env-file build-latest.env up --detach --no-build
 
       - name: Import keys
         working-directory: ./docker/localnet
@@ -89,16 +89,82 @@ jobs:
         working-directory: ./..
         run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
 
-      - name: Run tests
+      - name: Run Gingko integration tests
         working-directory: ./tests/integration
-        run: ginkgo -r --tags integration --race --randomize-suites --keep-going --trace --junit-report ../../report-integration.xml
+        run: ginkgo -r --race --randomize-suites --tags integration --keep-going --trace --junit-report ../../report-integration.xml
 
       - name: Upload integration tests result
         uses: actions/upload-artifact@v3
         with:
           name: report-integration.xml
           path: report-integration.xml
 
+  upgrade-migration-tests:
+    name: "Upgrade Integration Tests"
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      # Preparations
+
+      - name: Download old node binary (mainnet-latest)
+        run: |
+          mkdir -p ${{ env.RUNNER_BIN_DIR }}
+          wget -c https://github.com/cheqd/cheqd-node/releases/download/0.6.9/cheqd-noded-0.6.9-Linux-x86_64.tar.gz
+          tar -xvf cheqd-noded-0.6.9-Linux-x86_64.tar.gz -C ${{ env.RUNNER_BIN_DIR }} --strip-components=1
+          sudo chmod +x ${{ env.RUNNER_BIN_DIR }}/cheqd-noded
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version-file: ./go.mod
+          cache: true
+
+      - name: Install ginkgo
+        working-directory: ./..
+        run: go install github.com/onsi/ginkgo/v2/ginkgo@latest
+
+      - name: Download new version of the Docker image (build-latest)
+        uses: actions/download-artifact@v3
+        with:
+          name: cheqd-node-build.tar
+
+      - name: Load node Docker image
+        run: docker load -i cheqd-node-build.tar
+
+      # Run tests
+
+      - name: Setting up network with old binary inside (mainnet version)
+        working-directory: ./tests/upgrade/integration
+        run: |
+          bash scripts/setup.sh
+      
+      - name: Run pre-upgrade tests
+        working-directory: ./tests/upgrade
+        run: |
+          ginkgo -r --tags upgrade --race --tags upgrade_integration --focus-file pre_test.go
+        
+      - name: Restart network using new node version (build-latest)
+        working-directory: ./tests/upgrade/integration
+        run: |
+          bash scripts/upgrade.sh
+
+      - name: Run post-upgrade tests
+        working-directory: ./tests/upgrade
+        run: |
+          ginkgo -r --tags upgrade --race --tags upgrade_integration --focus-file post_test.go
+
+      - name: Cleanup after tests
+        working-directory: ./tests/upgrade/integration
+        run: |
+          bash scripts/cleanup.sh
+
   report-results:
     name: "Report"
     runs-on: ubuntu-latest
@@ -107,7 +173,6 @@ jobs:
 
     steps:
       - uses: actions/checkout@v3
-
       - name: Download unit tests report
         uses: actions/download-artifact@v3
         with:

diff --git a/README.md b/README.md
@@ -52,7 +52,7 @@ Once installed, `cheqd-node` can be controlled using the [cheqd Cosmos CLI guide
 
 ### 🔮 Upcoming functionality
 
-The [cheqd product roadmap](https://product.cheqd.io/updates/roadmap) describes in detail functionality that is being considered / in-development for future releases. A non-exhaustive list (not necessarily in order of priority) is:
+The [cheqd product roadmap](https://product.cheqd.io/product-essentials/roadmap) describes in detail functionality that is being considered / in-development for future releases. A non-exhaustive list (not necessarily in order of priority) is:
 
 * Revocation registry/list support to revoke issued credentials
 * Support for [AnonCreds](https://anoncreds-wg.github.io/anoncreds-spec/)

diff --git a/app/app.go b/app/app.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/cheqd/cheqd-node/app/ante"
+	"github.com/cheqd/cheqd-node/app/migrations"
 	appparams "github.com/cheqd/cheqd-node/app/params"
 	did "github.com/cheqd/cheqd-node/x/did"
 	didkeeper "github.com/cheqd/cheqd-node/x/did/keeper"
@@ -609,7 +610,30 @@ func New(
 	)
 
 	// Uncomment if you want to set a custom migration order here.
-	// app.mm.SetOrderMigrations(custom order)
+	app.mm.SetOrderMigrations(
+		upgradetypes.ModuleName,
+		capabilitytypes.ModuleName,
+		minttypes.ModuleName,
+		distrtypes.ModuleName,
+		slashingtypes.ModuleName,
+		evidencetypes.ModuleName,
+		stakingtypes.ModuleName,
+		authtypes.ModuleName,
+		banktypes.ModuleName,
+		govtypes.ModuleName,
+		crisistypes.ModuleName,
+		ibchost.ModuleName,
+		ibctransfertypes.ModuleName,
+		icatypes.ModuleName,
+		genutiltypes.ModuleName,
+		authz.ModuleName,
+		group.ModuleName,
+		feegrant.ModuleName,
+		paramstypes.ModuleName,
+		vestingtypes.ModuleName,
+		didtypes.ModuleName,
+		resourcetypes.ModuleName,
+	)
 
 	app.mm.RegisterInvariants(&app.CrisisKeeper)
 	app.mm.RegisterRoutes(app.Router(), app.QueryRouter(), encodingConfig.Amino)
@@ -644,18 +668,68 @@ func New(
 		func(ctx sdk.Context, _ upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
 			ctx.Logger().Info("Handler for upgrade plan: " + UpgradeName)
 
+			// Fix lack of version map initialization in InitChainer for new chains
+			if len(fromVM) == 0 {
+				println("Initializing version map")
+
+				// Add defaults for staking subspace
+				stakingSubspace, _ := app.ParamsKeeper.GetSubspace(stakingtypes.ModuleName)
+				stakingSubspace.Set(ctx, stakingtypes.KeyMinCommissionRate, sdk.NewDec(0))
+
+				// Fix version map
+				versionMap := app.mm.GetVersionMap()
+
+				for moduleName := range versionMap {
+					if _, ok := fromVM[moduleName]; !ok {
+						fromVM[moduleName] = versionMap[moduleName]
+					}
+				}
+			}
+
+			// cheqd migrations
+			migrationContext := migrations.NewMigrationContext(
+				app.appCodec,
+				keys[didtypes.StoreKey],
+				keys[resourcetypes.StoreKey])
+
+			cheqdMigrator := migrations.NewMigrator(
+				migrationContext,
+				[]migrations.Migration{
+					// Protobufs
+					migrations.MigrateDidProtobuf,
+					migrations.MigrateResourceProtobuf,
+
+					// Indy style
+					migrations.MigrateDidIndyStyle,
+					migrations.MigrateResourceIndyStyle,
+
+					// UUID normalizatiion
+					migrations.MigrateDidUUID,
+					migrations.MigrateResourceUUID,
+
+					// Did version id
+					migrations.MigrateDidVersionId,
+
+					// Resource checksum
+					migrations.MigrateResourceChecksum,
+
+					// Resource version links
+					migrations.MigrateResourceVersionLinks,
+
+					// Resource default alternative url
+					migrations.MigrateResourceDefaultAlternativeUrl,
+				})
+
+			err = cheqdMigrator.Migrate(ctx)
+			if err != nil {
+				panic(err)
+			}
+
 			// ibc v3 -> v4 migration
 			// transfer module consensus version has been bumped to 2
 			return app.mm.RunMigrations(ctx, app.configurator, fromVM)
 		})
 
-	// Test upgrade handler
-	app.UpgradeKeeper.SetUpgradeHandler(CosmovisorTestUpgrade, func(ctx sdk.Context, plan upgradetypes.Plan, fromVM module.VersionMap) (module.VersionMap, error) {
-		ctx.Logger().Info("Handler for upgrade plan: " + CosmovisorTestUpgrade)
-
-		return app.mm.RunMigrations(ctx, app.configurator, fromVM)
-	})
-
 	if loadLatest {
 		if err := app.LoadLatestVersion(); err != nil {
 			panic(err)

diff --git a/app/const.go b/app/const.go
@@ -4,6 +4,5 @@ const (
 	Name = "cheqd-node"
 	Home = ".cheqdnode"
 
-	UpgradeName           = "v1.0.0"
-	CosmovisorTestUpgrade = "cosmovisor_test"
+	UpgradeName = "v1.0.0"
 )
diff --git a/app/migrations/helpers/indy_style.go b/app/migrations/helpers/indy_style.go
@@ -0,0 +1,25 @@
+package helpers
+
+import (
+	"crypto/sha256"
+
+	didutils "github.com/cheqd/cheqd-node/x/did/utils"
+	"github.com/mr-tron/base58"
+)
+
+func MigrateIndyStyleDid(did string) string {
+	method, namespace, id := didutils.MustSplitDID(did)
+	return didutils.JoinDID(method, namespace, MigrateIndyStyleId(id))
+}
+
+func MigrateIndyStyleId(id string) string {
+	// If id is UUID it should not be changed
+	if didutils.IsValidUUID(id) {
+		return id
+	}
+
+	// Get Hash from current id to make a 32-symbol string
+	hash := sha256.Sum256([]byte(id))
+	// Indy-style identifier is 16-byte base58 string
+	return base58.Encode(hash[:16])
+}
diff --git a/app/migrations/helpers/indy_style_test.go b/app/migrations/helpers/indy_style_test.go
@@ -0,0 +1,17 @@
+package helpers
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+var _ = DescribeTable(
+	"Positive/Negative entries for checking indy style identifier compiling",
+	func(id string, outputStr string) {
+		Expect(MigrateIndyStyleId(id)).To(Equal(outputStr))
+	},
+
+	Entry("Valid: Real case: 16-symbol id", "zGqsJraNJCojDzG4", "QQHVWEaGae5Jts1quynR6M"),
+	Entry("Valid: Real case: 32-symbol id", "zGqsJraNJCojDzG4NXY2podMeaESVWvi", "AamcX5kPatrjccMNmuJxSo"),
+	Entry("Valid: UUID should not be changed", "F62542C3-4F71-4C21-8A2B-AD8DA460A976", "F62542C3-4F71-4C21-8A2B-AD8DA460A976"),
+)
diff --git a/app/migrations/helpers/storage.go b/app/migrations/helpers/storage.go
@@ -0,0 +1,28 @@
+package helpers
+
+import (
+	"github.com/cosmos/cosmos-sdk/store/types"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+type ByteStr []byte
+
+func ReadAllKeys(store types.KVStore, prefix []byte) []ByteStr {
+	keys := []ByteStr{}
+
+	iterator := sdk.KVStorePrefixIterator(store, prefix)
+	defer CloseIteratorOrPanic(iterator)
+
+	for ; iterator.Valid(); iterator.Next() {
+		keys = append(keys, ByteStr(iterator.Key()))
+	}
+
+	return keys
+}
+
+func CloseIteratorOrPanic(iterator sdk.Iterator) {
+	err := iterator.Close()
+	if err != nil {
+		panic(err.Error())
+	}
+}
diff --git a/app/migrations/helpers/suit_test.go b/app/migrations/helpers/suit_test.go
@@ -0,0 +1,13 @@
+package helpers
+
+import (
+	"testing"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+)
+
+func TestMigration(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "App Module Migration Suite")
+}
diff --git a/app/migrations/helpers/uuid.go b/app/migrations/helpers/uuid.go
@@ -0,0 +1,26 @@
+package helpers
+
+import (
+	didutils "github.com/cheqd/cheqd-node/x/did/utils"
+	"github.com/google/uuid"
+)
+
+func MigrateUUIDDid(did string) string {
+	method, namespace, id := didutils.MustSplitDID(did)
+	return didutils.JoinDID(method, namespace, MigrateUUIDId(id))
+}
+
+func MigrateUUIDId(id string) string {
+	// If id is not UUID it should not be changed
+	if !didutils.IsValidUUID(id) {
+		return id
+	}
+
+	// If uuid is already normalized, it should not be changed
+	if didutils.NormalizeUUID(id) == id {
+		return id
+	}
+
+	newId := uuid.NewSHA1(uuid.Nil, []byte(id))
+	return didutils.NormalizeUUID(newId.String())
+}