From adb503f4e2cdc658c6194c3d8409f055238ea927 Mon Sep 17 00:00:00 2001 From: Bogdan Kanivets Date: Thu, 1 Jun 2023 10:53:41 -0700 Subject: [PATCH] server: add fields to migrate from 3.5 to 3.4 Signed-off-by: Bogdan Kanivets --- etcdutl/etcdutl/migrate_command.go | 4 +- scripts/test.sh | 66 +++++-- server/storage/schema/bucket.go | 22 ++- server/storage/schema/confstate_test.go | 16 ++ server/storage/schema/membership.go | 2 +- server/storage/schema/membership_test.go | 46 +++++ server/storage/schema/schema.go | 40 +++- server/storage/schema/schema_test.go | 149 +++++++++------ tests/e2e/utl_migrate_test.go | 229 ++++++++++++++++++----- tests/framework/e2e/cluster.go | 3 + tests/framework/e2e/config.go | 1 + tests/framework/e2e/flags.go | 22 ++- 12 files changed, 446 insertions(+), 154 deletions(-) create mode 100644 server/storage/schema/membership_test.go diff --git a/etcdutl/etcdutl/migrate_command.go b/etcdutl/etcdutl/migrate_command.go index 521cf8ba80c8..4fc7f4644bfb 100644 --- a/etcdutl/etcdutl/migrate_command.go +++ b/etcdutl/etcdutl/migrate_command.go @@ -87,8 +87,8 @@ func (o *migrateOptions) Config() (*migrateConfig, error) { if err != nil { return nil, fmt.Errorf("failed to parse target version: %v", err) } - if c.targetVersion.LessThan(version.V3_5) { - return nil, fmt.Errorf(`target version %q not supported. Minimal "3.5"`, storageVersionToString(c.targetVersion)) + if c.targetVersion.LessThan(version.V3_4) { + return nil, fmt.Errorf(`target version %q not supported. Minimal "3.4"`, storageVersionToString(c.targetVersion)) } dbPath := datadir.ToBackendFileName(o.dataDir) diff --git a/scripts/test.sh b/scripts/test.sh index c8c6550ffa68..53fb5aeed922 100755 --- a/scripts/test.sh +++ b/scripts/test.sh @@ -565,44 +565,82 @@ function dep_pass { function release_pass { rm -f ./bin/etcd-last-release + rm -f ./bin/etcd-before-last-release + mkdir -p ./bin - # Work out the previous release based on the version reported by etcd binary + # Work out two previous releases based on the version reported by etcd binary binary_version=$(./bin/etcd --version | grep --only-matching --perl-regexp '(?<=etcd Version: )\d+\.\d+') binary_major=$(echo "${binary_version}" | cut -d '.' -f 1) binary_minor=$(echo "${binary_version}" | cut -d '.' -f 2) + previous_major=$binary_major previous_minor=$((binary_minor - 1)) + before_previous_major=$binary_major + before_previous_minor=$((binary_minor - 2)) # Handle the edge case where we go to a new major version # When this happens we obtain latest minor release of previous major if [ "${binary_minor}" -eq 0 ]; then - binary_major=$((binary_major - 1)) + previous_major=$((binary_major - 1)) + before_previous_major=$previous_minor previous_minor=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ | grep --only-matching --perl-regexp "(?<=v)${binary_major}.\d.[\d]+?(?=[\^])" \ | sort --numeric-sort --key 1.3 | tail -1 | cut -d '.' -f 2) + before_previous_minor=$((previous_minor - 1)) + fi + + # Handle the edge case when only 'before previous' should be latest minor release of previous major + if [ "${binary_minor}" -eq 1 ]; then + before_previous_major=$((binary_major - 1)) + before_previous_minor=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ + | grep --only-matching --perl-regexp "(?<=v)${before_previous_major}.\d.[\d]+?(?=[\^])" \ + | sort --numeric-sort --key 1.3 | tail -1 | cut -d '.' -f 2) fi # This gets a list of all remote tags for the release branch in regex # Sort key is used to sort numerically by patch version # Latest version is then stored for use below - UPGRADE_VER=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ - | grep --only-matching --perl-regexp "(?<=v)${binary_major}.${previous_minor}.[\d]+?(?=[\^])" \ + UPGRADE_VER_LAST=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ + | grep --only-matching --perl-regexp "(?<=v)${previous_major}.${previous_minor}.[\d]+?(?=[\^])" \ + | sort --numeric-sort --key 1.5 | tail -1 | sed 's/^/v/') + log_callout "Found latest release: ${UPGRADE_VER_LAST}." + + if [ -n "${MANUAL_VER_LAST:-}" ]; then + # in case, we need to test against different version + UPGRADE_VER_LAST=$MANUAL_VER_LAST + fi + if [[ -z ${UPGRADE_VER_LAST} ]]; then + UPGRADE_VER_LAST="v3.5.0" + log_warning "fallback to" ${UPGRADE_VER_LAST} + fi + + UPGRADE_VER_BEFORE_LAST=$(git ls-remote --tags https://github.com/etcd-io/etcd.git \ + | grep --only-matching --perl-regexp "(?<=v)${before_previous_major}.${before_previous_minor}.[\d]+?(?=[\^])" \ | sort --numeric-sort --key 1.5 | tail -1 | sed 's/^/v/') - log_callout "Found latest release: ${UPGRADE_VER}." + log_callout "Found before latest release: ${UPGRADE_VER_BEFORE_LAST}." - if [ -n "${MANUAL_VER:-}" ]; then + if [ -n "${MANUAL_VER_BEFORE_LAST:-}" ]; then # in case, we need to test against different version - UPGRADE_VER=$MANUAL_VER + UPGRADE_VER_BEFORE_LAST=MANUAL_VER_BEFORE_LAST fi - if [[ -z ${UPGRADE_VER} ]]; then - UPGRADE_VER="v3.5.0" - log_warning "fallback to" ${UPGRADE_VER} + if [[ -z ${UPGRADE_VER_BEFORE_LAST} ]]; then + UPGRADE_VER_BEFORE_LAST="v3.4.0" + log_warning "fallback to" ${UPGRADE_VER_BEFORE_LAST} fi - local file="etcd-$UPGRADE_VER-linux-$GOARCH.tar.gz" - log_callout "Downloading $file" + download_etcd_ver_to_tmp ${UPGRADE_VER_LAST} + mv /tmp/etcd ./bin/etcd-last-release + + download_etcd_ver_to_tmp ${UPGRADE_VER_BEFORE_LAST} + mv /tmp/etcd ./bin/etcd-before-last-release +} + +function download_etcd_ver_to_tmp { + local version="$1" + local file="etcd-$version-linux-$GOARCH.tar.gz" + log_callout "Downloading $file" set +e - curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$UPGRADE_VER/$file" -o "/tmp/$file" + curl --fail -L "https://github.com/etcd-io/etcd/releases/download/$version/$file" -o "/tmp/$file" local result=$? set -e case $result in @@ -613,8 +651,6 @@ function release_pass { esac tar xzvf "/tmp/$file" -C /tmp/ --strip-components=1 - mkdir -p ./bin - mv /tmp/etcd ./bin/etcd-last-release } function mod_tidy_for_module { diff --git a/server/storage/schema/bucket.go b/server/storage/schema/bucket.go index 5472af3c3b47..30d44add8940 100644 --- a/server/storage/schema/bucket.go +++ b/server/storage/schema/bucket.go @@ -68,17 +68,19 @@ func (b bucket) String() string { return string(b.Name()) } func (b bucket) IsSafeRangeBucket() bool { return b.safeRangeBucket } var ( - // Pre v3.5 - ScheduledCompactKeyName = []byte("scheduledCompactRev") - FinishedCompactKeyName = []byte("finishedCompactRev") - MetaConsistentIndexKeyName = []byte("consistent_index") - AuthEnabledKeyName = []byte("authEnabled") - AuthRevisionKeyName = []byte("authRevision") - // Since v3.5 - MetaTermKeyName = []byte("term") - MetaConfStateName = []byte("confState") + // Pre v3.4 + ScheduledCompactKeyName = []byte("scheduledCompactRev") + FinishedCompactKeyName = []byte("finishedCompactRev") + MetaConsistentIndexKeyName = []byte("consistent_index") + AuthEnabledKeyName = []byte("authEnabled") + AuthRevisionKeyName = []byte("authRevision") ClusterClusterVersionKeyName = []byte("clusterVersion") - ClusterDowngradeKeyName = []byte("downgrade") + + // No new keys were added in v3.4 + // Since v3.5 + MetaTermKeyName = []byte("term") + MetaConfStateName = []byte("confState") + ClusterDowngradeKeyName = []byte("downgrade") // Since v3.6 MetaStorageVersionName = []byte("storageVersion") // Before adding new meta key please update server/etcdserver/version diff --git a/server/storage/schema/confstate_test.go b/server/storage/schema/confstate_test.go index 11131e40579c..471a913202e8 100644 --- a/server/storage/schema/confstate_test.go +++ b/server/storage/schema/confstate_test.go @@ -59,6 +59,22 @@ func TestMustUnsafeSaveConfStateToBackend(t *testing.T) { assert.Nil(t, UnsafeConfStateFromBackend(lg, tx)) }) + emptyConfState := raftpb.ConfState{} + t.Run("save empty", func(t *testing.T) { + tx := be.BatchTx() + tx.Lock() + MustUnsafeSaveConfStateToBackend(lg, tx, &emptyConfState) + tx.Unlock() + tx.Commit() + }) + + t.Run("read empty", func(t *testing.T) { + tx := be.ReadTx() + tx.RLock() + defer tx.RUnlock() + assert.Equal(t, emptyConfState, *UnsafeConfStateFromBackend(lg, tx)) + }) + confState := raftpb.ConfState{Learners: []uint64{1, 2}, Voters: []uint64{3}, AutoLeave: false} t.Run("save", func(t *testing.T) { diff --git a/server/storage/schema/membership.go b/server/storage/schema/membership.go index 137d27850717..5a153003d62f 100644 --- a/server/storage/schema/membership.go +++ b/server/storage/schema/membership.go @@ -183,7 +183,7 @@ func mustParseMemberIDFromBytes(lg *zap.Logger, key []byte) types.ID { } // ClusterVersionFromBackend reads cluster version from backend. -// The field is populated since etcd v3.5. +// The field is populated since etcd v3.4. func (s *membershipBackend) ClusterVersionFromBackend() *semver.Version { ckey := ClusterClusterVersionKeyName tx := s.be.ReadTx() diff --git a/server/storage/schema/membership_test.go b/server/storage/schema/membership_test.go new file mode 100644 index 000000000000..e1c2afffc4eb --- /dev/null +++ b/server/storage/schema/membership_test.go @@ -0,0 +1,46 @@ +// Copyright 2023 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package schema + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "go.uber.org/zap/zaptest" + + "go.etcd.io/etcd/api/v3/version" + serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" + betesting "go.etcd.io/etcd/server/v3/storage/backend/testing" +) + +func TestDowngradeInfoFromBackend(t *testing.T) { + lg := zaptest.NewLogger(t) + be, _ := betesting.NewDefaultTmpBackend(t) + defer betesting.Close(t, be) + + mbe := NewMembershipBackend(lg, be) + + mbe.MustCreateBackendBuckets() + mbe.be.ForceCommit() + assert.Nil(t, mbe.DowngradeInfoFromBackend()) + + dinfo := &serverversion.DowngradeInfo{Enabled: true, TargetVersion: version.V3_5.String()} + mbe.MustSaveDowngradeToBackend(dinfo) + + info := mbe.DowngradeInfoFromBackend() + + assert.Equal(t, dinfo, info) +} diff --git a/server/storage/schema/schema.go b/server/storage/schema/schema.go index 890810511ae7..af238fb916be 100644 --- a/server/storage/schema/schema.go +++ b/server/storage/schema/schema.go @@ -15,14 +15,17 @@ package schema import ( + "encoding/json" "fmt" "github.com/coreos/go-semver/semver" + "go.uber.org/zap" "go.etcd.io/etcd/api/v3/version" - + serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/storage/backend" + "go.etcd.io/raft/v3/raftpb" ) // Validate checks provided backend to confirm that schema used is supported. @@ -83,7 +86,7 @@ func UnsafeMigrate(lg *zap.Logger, tx backend.UnsafeReadWriter, w WALVersion, ta // DetectSchemaVersion returns version of storage schema. Returned value depends on etcd version that created the backend. For // * v3.6 and newer will return storage version. // * v3.5 will return it's version if it includes all storage fields added in v3.5 (might require a snapshot). -// * v3.4 and older is not supported and will return error. +// * v3.4 will return it's version if it doesn't include all storage fields added in v3.5. func DetectSchemaVersion(lg *zap.Logger, tx backend.ReadTx) (v semver.Version, err error) { tx.RLock() defer tx.RUnlock() @@ -96,12 +99,18 @@ func UnsafeDetectSchemaVersion(lg *zap.Logger, tx backend.UnsafeReader) (v semve if vp != nil { return *vp, nil } + confstate := UnsafeConfStateFromBackend(lg, tx) - if confstate == nil { - return v, fmt.Errorf("missing confstate information") - } _, term := UnsafeReadConsistentIndex(tx) - if term == 0 { + if confstate == nil && term == 0 { + // if both confstate and term are missing, assume it's v3.4 + return version.V3_4, nil + } else if confstate == nil { + return v, fmt.Errorf("missing confstate information") + } else if len(confstate.Voters) == 0 && term == 0 { + // if confstate is empty and term is missing, assume it's v3.5 that was migrated from v3.4 and never started + return version.V3_5, nil + } else if term == 0 { return v, fmt.Errorf("missing term information") } return version.V3_5, nil @@ -129,10 +138,21 @@ var ( // schema was introduced in v3.6 as so its changes were not tracked before. schemaChanges = map[semver.Version][]schemaChange{ version.V3_6: { - addNewField(Meta, MetaStorageVersionName, emptyStorageVersion), + // emptyValue is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator. + addNewField(Meta, MetaStorageVersionName, emptyValue), + }, + version.V3_5: { + // UnsafeReadConsistentIndex will fail on []byte(""), use 0 as default + addNewField(Meta, MetaTermKeyName, emptyTerm), + // UnsafeConfStateFromBackend will fail on []byte(""), use empty struct as default + addNewField(Meta, MetaConfStateName, emptyConfState), + // DowngradeInfoFromBackend will fail on []byte(""), false is better default + addNewField(Cluster, ClusterDowngradeKeyName, falseDowngradeInfo), }, } - // emptyStorageVersion is used for v3.6 Step for the first time, in all other version StoragetVersion should be set by migrator. - // Adding a addNewField for StorageVersion we can reuse logic to remove it when downgrading to v3.5 - emptyStorageVersion = []byte("") + + emptyValue = []byte("") + emptyTerm = make([]byte, 8) + emptyConfState, _ = json.Marshal(raftpb.ConfState{}) + falseDowngradeInfo, _ = json.Marshal(serverversion.DowngradeInfo{Enabled: false, TargetVersion: ""}) ) diff --git a/server/storage/schema/schema_test.go b/server/storage/schema/schema_test.go index 3d9bd1065c3a..ebdea74dbbf5 100644 --- a/server/storage/schema/schema_test.go +++ b/server/storage/schema/schema_test.go @@ -105,99 +105,115 @@ func TestMigrate(t *testing.T) { targetVersion semver.Version walEntries []etcdserverpb.InternalRaftRequest - expectVersion *semver.Version - expectError bool - expectErrorMsg string + expectSchemaVersion semver.Version + expectedStorageVersion *semver.Version + expectError bool + expectErrorMsg string }{ // As storage version field was added in v3.6, for v3.5 we will not set it. // For storage to be considered v3.5 it have both confstate and term key set. { - name: `Upgrading v3.5 to v3.6 should be rejected if confstate is not set`, - version: version.V3_5, - overrideKeys: func(tx backend.UnsafeReadWriter) {}, - targetVersion: version.V3_6, - expectVersion: nil, - expectError: true, - expectErrorMsg: `cannot detect storage schema version: missing confstate information`, + name: `Upgrading v3.5 to v3.6 should be rejected if confstate is not set`, + version: version.V3_5, + overrideKeys: func(tx backend.UnsafeReadWriter) { + UnsafeUpdateConsistentIndex(tx, 1, 1) + }, + targetVersion: version.V3_6, + expectedStorageVersion: nil, + expectError: true, + expectErrorMsg: `cannot detect storage schema version: missing confstate information`, }, { name: `Upgrading v3.5 to v3.6 should be rejected if term is not set`, version: version.V3_5, overrideKeys: func(tx backend.UnsafeReadWriter) { - MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{}) + MustUnsafeSaveConfStateToBackend(zap.NewNop(), tx, &raftpb.ConfState{Voters: []uint64{1}}) }, - targetVersion: version.V3_6, - expectVersion: nil, - expectError: true, - expectErrorMsg: `cannot detect storage schema version: missing term information`, + targetVersion: version.V3_6, + expectedStorageVersion: nil, + expectError: true, + expectErrorMsg: `cannot detect storage schema version: missing term information`, }, { - name: `Upgrading v3.5 to v3.6 should succeed; all required fields are set`, - version: version.V3_5, - targetVersion: version.V3_6, - expectVersion: &version.V3_6, + name: `Upgrading v3.5 to v3.6 should succeed; all required fields are set`, + version: version.V3_5, + targetVersion: version.V3_6, + expectSchemaVersion: version.V3_6, + expectedStorageVersion: &version.V3_6, }, { - name: `Migrate on same v3.5 version passes and doesn't set storage version'`, - version: version.V3_5, - targetVersion: version.V3_5, - expectVersion: nil, + name: `Migrate on same v3.5 version passes and doesn't set storage version'`, + version: version.V3_5, + targetVersion: version.V3_5, + expectSchemaVersion: version.V3_5, + expectedStorageVersion: nil, }, { - name: `Migrate on same v3.6 version passes`, - version: version.V3_6, - targetVersion: version.V3_6, - expectVersion: &version.V3_6, + name: `Migrate on same v3.6 version passes`, + version: version.V3_6, + targetVersion: version.V3_6, + expectSchemaVersion: version.V3_6, + expectedStorageVersion: &version.V3_6, }, { - name: `Migrate on same v3.7 version passes`, - version: version.V3_7, - targetVersion: version.V3_7, - expectVersion: &version.V3_7, + name: `Migrate on same v3.7 version passes`, + version: version.V3_7, + targetVersion: version.V3_7, + expectSchemaVersion: version.V3_7, + expectedStorageVersion: &version.V3_7, }, { - name: "Upgrading 3.6 to v3.7 is not supported", - version: version.V3_6, - targetVersion: version.V3_7, - expectVersion: &version.V3_6, - expectError: true, - expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`, + name: "Upgrading 3.6 to v3.7 is not supported", + version: version.V3_6, + targetVersion: version.V3_7, + expectedStorageVersion: &version.V3_6, + expectError: true, + expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`, }, { - name: "Downgrading v3.7 to v3.6 is not supported", - version: version.V3_7, - targetVersion: version.V3_6, - expectVersion: &version.V3_7, - expectError: true, - expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`, + name: "Downgrading v3.7 to v3.6 is not supported", + version: version.V3_7, + targetVersion: version.V3_6, + expectedStorageVersion: &version.V3_7, + expectError: true, + expectErrorMsg: `cannot create migration plan: version "3.7.0" is not supported`, }, { - name: "Downgrading v3.6 to v3.5 works as there are no v3.6 wal entries", - version: version.V3_6, - targetVersion: version.V3_5, + name: "Downgrading v3.6 to v3.5 works as there are no v3.6 wal entries", + version: version.V3_6, + targetVersion: version.V3_5, + expectSchemaVersion: version.V3_5, walEntries: []etcdserverpb.InternalRaftRequest{ {Range: &etcdserverpb.RangeRequest{Key: []byte("\x00"), RangeEnd: []byte("\xff")}}, }, - expectVersion: nil, + expectedStorageVersion: nil, }, { - name: "Downgrading v3.6 to v3.5 fails if there are newer WAL entries", - version: version.V3_6, - targetVersion: version.V3_5, + name: "Downgrading v3.6 to v3.5 fails if there are newer WAL entries", + version: version.V3_6, + targetVersion: version.V3_5, + expectSchemaVersion: version.V3_5, walEntries: []etcdserverpb.InternalRaftRequest{ {ClusterVersionSet: &membershippb.ClusterVersionSetRequest{Ver: "3.6.0"}}, }, - expectVersion: &version.V3_6, - expectError: true, - expectErrorMsg: "cannot downgrade storage, WAL contains newer entries", + expectedStorageVersion: &version.V3_6, + expectError: true, + expectErrorMsg: "cannot downgrade storage, WAL contains newer entries", }, { - name: "Downgrading v3.5 to v3.4 is not supported as schema was introduced in v3.6", - version: version.V3_5, - targetVersion: version.V3_4, - expectVersion: nil, - expectError: true, - expectErrorMsg: `cannot create migration plan: version "3.5.0" is not supported`, + name: "Downgrading v3.5 to v3.4 is supported", + version: version.V3_5, + targetVersion: version.V3_4, + expectSchemaVersion: version.V3_4, + // note: 3.4 doesn't have storageVersion, this field was added in 3.6 + expectedStorageVersion: nil, + }, + { + name: `Upgrading v3.4 to v3.5 should succeed`, + version: version.V3_4, + targetVersion: version.V3_5, + expectSchemaVersion: version.V3_5, + expectedStorageVersion: nil, }, } for _, tc := range tcs { @@ -221,8 +237,18 @@ func TestMigrate(t *testing.T) { if err != nil && err.Error() != tc.expectErrorMsg { t.Errorf("Migrate(lg, tx, %q) = %q, expected error message: %q", tc.targetVersion, err, tc.expectErrorMsg) } - v := UnsafeReadStorageVersion(b.BatchTx()) - assert.Equal(t, tc.expectVersion, v) + b.ForceCommit() + + storeV := UnsafeReadStorageVersion(b.BatchTx()) + assert.Equal(t, tc.expectedStorageVersion, storeV) + + if !tc.expectError { + v, err := DetectSchemaVersion(lg, b.ReadTx()) + if err != nil { + t.Errorf("Migrate(lg, tx, %q) failed DetectSchemaVersion: %q", tc.targetVersion, err) + } + assert.Equal(t, tc.expectSchemaVersion, v) + } }) } } @@ -303,6 +329,7 @@ func setupBackendData(t *testing.T, ver semver.Version, overrideKeys func(tx bac } tx.Lock() UnsafeCreateMetaBucket(tx) + tx.UnsafeCreateBucket(Cluster) if overrideKeys != nil { overrideKeys(tx) } else { diff --git a/tests/e2e/utl_migrate_test.go b/tests/e2e/utl_migrate_test.go index a58d69a24c74..1e471c7f0529 100644 --- a/tests/e2e/utl_migrate_test.go +++ b/tests/e2e/utl_migrate_test.go @@ -17,6 +17,7 @@ package e2e import ( "context" "fmt" + "os" "path/filepath" "strings" "testing" @@ -35,88 +36,114 @@ import ( "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestEtctlutlMigrate(t *testing.T) { +type bucketKey struct { + bucket backend.Bucket + key []byte +} + +func TestEtctlutlMigrateSuccess(t *testing.T) { lastReleaseBinary := e2e.BinPath.EtcdLastRelease + beforeLastReleaseBinary := e2e.BinPath.EtcdBeforeLastRelease tcs := []struct { - name string - targetVersion string - clusterVersion e2e.ClusterVersion - force bool + name string + targetVersion string + clusterVersion e2e.ClusterVersion + targetClusterVersion e2e.ClusterVersion + force bool + cleanWal bool expectLogsSubString string expectStorageVersion *semver.Version + + expectNonFoundKeys []bucketKey + expectStartError string }{ { - name: "Invalid target version string", - targetVersion: "abc", - expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "abc"`, - expectStorageVersion: &version.V3_6, - }, - { - name: "Invalid target version", - targetVersion: "3.a", - expectLogsSubString: `Error: failed to parse target version: strconv.ParseInt: parsing "a": invalid syntax`, - expectStorageVersion: &version.V3_6, - }, - { - name: "Target with only major version is invalid", - targetVersion: "3", - expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3"`, - expectStorageVersion: &version.V3_6, + name: "Upgrade v3.4 to v3.5 should work", + clusterVersion: e2e.BeforeLastVersion, + targetVersion: "3.5", + targetClusterVersion: e2e.LastVersion, }, + { - name: "Target with patch version is invalid", - targetVersion: "3.6.0", - expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3.6.0"`, - expectStorageVersion: &version.V3_6, + name: "Downgrade v3.5 to v3.4 should work", + clusterVersion: e2e.LastVersion, + targetVersion: "3.4", + targetClusterVersion: e2e.BeforeLastVersion, + cleanWal: true, + expectLogsSubString: "updated storage version\t" + `{"new-storage-version": "3.4.0"}`, + expectNonFoundKeys: []bucketKey{ + {bucket: schema.Meta, key: schema.MetaTermKeyName}, + {bucket: schema.Meta, key: schema.MetaConfStateName}, + {bucket: schema.Cluster, key: schema.ClusterDowngradeKeyName}, + }, }, { - name: "Migrate v3.5 to v3.5 is no-op", - clusterVersion: e2e.LastVersion, - targetVersion: "3.5", - expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.5"}`, + name: "Migrate v3.5 to v3.5 is no-op", + clusterVersion: e2e.LastVersion, + targetVersion: "3.5", + targetClusterVersion: e2e.LastVersion, + expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.5"}`, }, { name: "Upgrade v3.5 to v3.6 should work", clusterVersion: e2e.LastVersion, targetVersion: "3.6", + targetClusterVersion: e2e.CurrentVersion, expectStorageVersion: &version.V3_6, }, { name: "Migrate v3.6 to v3.6 is no-op", + clusterVersion: e2e.CurrentVersion, targetVersion: "3.6", + targetClusterVersion: e2e.CurrentVersion, expectLogsSubString: "storage version up-to-date\t" + `{"storage-version": "3.6"}`, expectStorageVersion: &version.V3_6, }, { name: "Downgrade v3.6 to v3.5 should fail until it's implemented", + clusterVersion: e2e.CurrentVersion, targetVersion: "3.5", + targetClusterVersion: e2e.CurrentVersion, expectLogsSubString: "cannot downgrade storage, WAL contains newer entries", expectStorageVersion: &version.V3_6, }, { - name: "Downgrade v3.6 to v3.5 with force should work", - targetVersion: "3.5", - force: true, - expectLogsSubString: "forcefully cleared storage version", + name: "Downgrade v3.6 to v3.5 should work when wal file doesn't have version entry", + clusterVersion: e2e.CurrentVersion, + targetVersion: "3.5", + targetClusterVersion: e2e.LastVersion, + cleanWal: true, + expectLogsSubString: "updated storage version\t" + `{"new-storage-version": "3.5.0"}`, + expectNonFoundKeys: []bucketKey{ + {bucket: schema.Meta, key: schema.MetaStorageVersionName}, + }, }, { name: "Upgrade v3.6 to v3.7 with force should work", + clusterVersion: e2e.CurrentVersion, targetVersion: "3.7", force: true, expectLogsSubString: "forcefully set storage version\t" + `{"storage-version": "3.7"}`, expectStorageVersion: &semver.Version{Major: 3, Minor: 7}, + // we don't have 3.7 binary, try to start with existing and verify error + targetClusterVersion: e2e.CurrentVersion, + expectStartError: "version \\\"3.7.0\\\" is not supported", }, } for _, tc := range tcs { t.Run(tc.name, func(t *testing.T) { e2e.BeforeTest(t) lg := zaptest.NewLogger(t) - if tc.clusterVersion != e2e.CurrentVersion && !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { + if tc.clusterVersion == e2e.LastVersion && !fileutil.Exist(e2e.BinPath.EtcdLastRelease) { t.Skipf("%q does not exist", lastReleaseBinary) } + if tc.clusterVersion == e2e.BeforeLastVersion && !fileutil.Exist(e2e.BinPath.EtcdBeforeLastRelease) { + t.Skipf("%q does not exist", beforeLastReleaseBinary) + } dataDirPath := t.TempDir() + defer os.RemoveAll(dataDirPath) epc, err := e2e.NewEtcdProcessCluster(context.TODO(), t, e2e.WithVersion(tc.clusterVersion), @@ -129,29 +156,54 @@ func TestEtctlutlMigrate(t *testing.T) { if err != nil { t.Fatalf("could not start etcd process cluster (%v)", err) } - defer func() { - if errC := epc.Close(); errC != nil { - t.Fatalf("error closing etcd processes (%v)", errC) - } - }() dialTimeout := 10 * time.Second prefixArgs := []string{e2e.BinPath.Etcdctl, "--endpoints", strings.Join(epc.EndpointsGRPC(), ","), "--dial-timeout", dialTimeout.String()} - t.Log("Write keys to ensure wal snapshot is created and all v3.5 fields are set...") + t.Log("Write keys to ensure wal snapshot is created and all fields are set...") for i := 0; i < 10; i++ { if err = e2e.SpawnWithExpect(append(prefixArgs, "put", fmt.Sprintf("%d", i), "value"), expect.ExpectedResponse{Value: "OK"}); err != nil { t.Fatal(err) } } + memberDataDir := epc.Procs[0].Config().DataDirPath + if tc.cleanWal { + // migrate command doesn't handle version entries in wal file + // we'll get an error when starting etcd: "etcdserver/membership: cluster cannot be downgraded (current version: X is lower than determined cluster version: X)" + // snapshot restore will generate a clean wal file that doesn't have version entries + + t.Log("Taking snapshot...") + spath := filepath.Join(dataDirPath, "snapshot") + cmdArgs := append(prefixArgs, "snapshot", "save", spath) + err = e2e.SpawnWithExpect(cmdArgs, expect.ExpectedResponse{Value: fmt.Sprintf("Snapshot saved at %s", spath)}) + if err != nil { + t.Fatal(err) + } + + t.Log("Restoring snapshot...") + dataDirPath = filepath.Join(dataDirPath, "restored") + memberDataDir = filepath.Join(dataDirPath, "member-0") + t.Log("etcdctl restoring the snapshot...") + err = e2e.SpawnWithExpect([]string{ + e2e.BinPath.Etcdutl, "snapshot", "restore", spath, + "--name", epc.Procs[0].Config().Name, + "--initial-cluster", epc.Procs[0].Config().InitialCluster, + "--initial-cluster-token", epc.Procs[0].Config().InitialToken, + "--initial-advertise-peer-urls", epc.Procs[0].Config().PeerURL.String(), + "--data-dir", memberDataDir}, + expect.ExpectedResponse{Value: "added member"}) + if err != nil { + t.Fatal(err) + } + } + t.Log("Stopping the server...") - if err = epc.Procs[0].Stop(); err != nil { - t.Fatal(err) + if errC := epc.Close(); errC != nil { + t.Fatalf("error closing etcd processes (%v)", errC) } t.Log("etcdutl migrate...") - memberDataDir := epc.Procs[0].Config().DataDirPath args := []string{e2e.BinPath.Etcdutl, "migrate", "--data-dir", memberDataDir, "--target-version", tc.targetVersion} if tc.force { args = append(args, "--force") @@ -165,12 +217,99 @@ func TestEtctlutlMigrate(t *testing.T) { } } - t.Log("etcdutl migrate...") + t.Log("verify backend...") be := backend.NewDefaultBackend(lg, filepath.Join(memberDataDir, "member/snap/db")) - defer be.Close() ver := schema.ReadStorageVersion(be.ReadTx()) assert.Equal(t, tc.expectStorageVersion, ver) + + for _, bk := range tc.expectNonFoundKeys { + _, vs := be.ReadTx().UnsafeRange(bk.bucket, bk.key, nil, 0) + assert.Zero(t, len(vs)) + } + + be.Close() + + t.Log("Starting target version server...") + epct, err := e2e.NewEtcdProcessCluster(context.TODO(), t, + e2e.WithVersion(tc.targetClusterVersion), + e2e.WithDataDirPath(dataDirPath), + e2e.WithClusterSize(1), + e2e.WithKeepDataDir(true), + // Set low SnapshotCount to ensure wal snapshot is done + e2e.WithSnapshotCount(1), + ) + + if err != nil { + if tc.expectStartError != "" && strings.Contains(err.Error(), tc.expectStartError) { + t.Logf("Found expected start error: %s", tc.expectStartError) + return + } + t.Fatalf("could not start etcd process cluster (%v)", err) + } + + defer func() { + if errC := epct.Close(); errC != nil { + t.Fatalf("error closing etcd processes (%v)", errC) + } + }() + + t.Log("Read keys ...") + for i := 0; i < 10; i++ { + if err = e2e.SpawnWithExpect(append(prefixArgs, "get", fmt.Sprintf("%d", i)), expect.ExpectedResponse{Value: "value"}); err != nil { + t.Fatal(err) + } + } + + }) + } +} + +func TestEtctlutlMigrateError(t *testing.T) { + tcs := []struct { + name string + targetVersion string + clusterVersion e2e.ClusterVersion + force bool + + expectLogsSubString string + expectStorageVersion *semver.Version + }{ + { + name: "Invalid target version string", + targetVersion: "abc", + expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "abc"`, + }, + { + name: "Invalid target version", + targetVersion: "3.a", + expectLogsSubString: `Error: failed to parse target version: strconv.ParseInt: parsing "a": invalid syntax`, + }, + { + name: "Target with only major version is invalid", + targetVersion: "3", + expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3"`, + }, + { + name: "Target with patch version is invalid", + targetVersion: "3.6.0", + expectLogsSubString: `Error: wrong target version format, expected "X.Y", got "3.6.0"`, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + e2e.BeforeTest(t) + + t.Log("etcdutl migrate...") + args := []string{e2e.BinPath.Etcdutl, "migrate", "--data-dir", t.TempDir(), "--target-version", tc.targetVersion} + err := e2e.SpawnWithExpect(args, expect.ExpectedResponse{Value: tc.expectLogsSubString}) + if err != nil { + if tc.expectLogsSubString != "" { + require.ErrorContains(t, err, tc.expectLogsSubString) + } else { + t.Fatal(err) + } + } }) } } diff --git a/tests/framework/e2e/cluster.go b/tests/framework/e2e/cluster.go index 17c3c37a0d3e..00eddf95b7c0 100644 --- a/tests/framework/e2e/cluster.go +++ b/tests/framework/e2e/cluster.go @@ -643,6 +643,9 @@ func (cfg *EtcdProcessClusterConfig) EtcdServerProcessConfig(tb testing.TB, i in } case LastVersion: execPath = BinPath.EtcdLastRelease + + case BeforeLastVersion: + execPath = BinPath.EtcdBeforeLastRelease default: panic(fmt.Sprintf("Unknown cluster version %v", cfg.Version)) } diff --git a/tests/framework/e2e/config.go b/tests/framework/e2e/config.go index acc1d82e0484..6fca2f7bcb28 100644 --- a/tests/framework/e2e/config.go +++ b/tests/framework/e2e/config.go @@ -21,6 +21,7 @@ const ( MinorityLastVersion ClusterVersion = "minority-last-version" QuorumLastVersion ClusterVersion = "quorum-last-version" LastVersion ClusterVersion = "last-version" + BeforeLastVersion ClusterVersion = "before-last-version" ) type ClusterContext struct { diff --git a/tests/framework/e2e/flags.go b/tests/framework/e2e/flags.go index 23bdde6590ce..a3e0565917cf 100644 --- a/tests/framework/e2e/flags.go +++ b/tests/framework/e2e/flags.go @@ -44,11 +44,12 @@ var ( ) type binPath struct { - Etcd string - EtcdLastRelease string - Etcdctl string - Etcdutl string - LazyFS string + Etcd string + EtcdLastRelease string + EtcdBeforeLastRelease string + Etcdctl string + Etcdutl string + LazyFS string } func (bp *binPath) LazyFSAvailable() bool { @@ -73,11 +74,12 @@ func InitFlags() { flag.Parse() BinPath = binPath{ - Etcd: *binDir + "/etcd", - EtcdLastRelease: *binDir + "/etcd-last-release", - Etcdctl: *binDir + "/etcdctl", - Etcdutl: *binDir + "/etcdutl", - LazyFS: *binDir + "/lazyfs", + Etcd: *binDir + "/etcd", + EtcdLastRelease: *binDir + "/etcd-last-release", + EtcdBeforeLastRelease: *binDir + "/etcd-before-last-release", + Etcdctl: *binDir + "/etcdctl", + Etcdutl: *binDir + "/etcdutl", + LazyFS: *binDir + "/lazyfs", } CertPath = CertDir + "/server.crt" PrivateKeyPath = CertDir + "/server.key.insecure"