tests: Add e2e tests for downgrades

tests/e2e/cluster_downgrade_test.go

@@ -0,0 +1,134 @@
+// Copyright 2021 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"testing"
+	"time"
+
+	"go.etcd.io/etcd/api/v3/version"
+	clientv3 "go.etcd.io/etcd/client/v3"
+)
+
+func TestDowngradeUpgrade(t *testing.T) {
+	currentEtcdBinary := ""
+	lastReleaseBinary := binDir + "/etcd-last-release"
+
+	BeforeTest(t)
+	dataDirPath := t.TempDir()
+
+	epc := startEtcd(t, currentEtcdBinary, dataDirPath)
+	validateVersion(t, epc, version.Versions{Cluster: "3.6", Server: "3.6"})
+
+	downgradeEnable(t, epc)
+	validateVersion(t, epc, version.Versions{Cluster: "3.5", Server: "3.6"})
+
+	stopEtcd(t, epc)
+	epc = startEtcd(t, lastReleaseBinary, dataDirPath)
+	validateVersion(t, epc, version.Versions{Cluster: "3.5", Server: "3.5"})
+	expectLog(t, epc, "the cluster has been downgraded")
+
+	stopEtcd(t, epc)
+	epc = startEtcd(t, currentEtcdBinary, dataDirPath)
+	// TODO: Verify cluster version after upgrade when we fix cluster version set timeout
+	validateVersion(t, epc, version.Versions{Server: "3.6"})
+}
+
+func startEtcd(t *testing.T, execPath, dataDirPath string) *etcdProcessCluster {
+	epc, err := newEtcdProcessCluster(t, &etcdProcessClusterConfig{
+		execPath:     execPath,
+		dataDirPath:  dataDirPath,
+		clusterSize:  1,
+		initialToken: "new",
+		keepDataDir:  true,
+		// TODO: REMOVE snapshot override when snapshotting is automated after lowering storage versiont l
+		snapshotCount: 5,
+	})
+	if err != nil {
+		t.Fatalf("could not start etcd process cluster (%v)", err)
+	}
+	t.Cleanup(func() {
+		if errC := epc.Close(); errC != nil {
+			t.Fatalf("error closing etcd processes (%v)", errC)
+		}
+	})
+
+	prefixArgs := []string{ctlBinPath, "--endpoints", strings.Join(epc.EndpointsV3(), ",")}
+	t.Log("Write keys to ensure wal snapshot is created so cluster version set is snapshotted")
+	executeWithTimeout(t, 20*time.Second, func() {
+		for i := 0; i < 10; i++ {
+			if err := spawnWithExpect(append(prefixArgs, "put", fmt.Sprintf("%d", i), "value"), "OK"); err != nil {
+				t.Fatal(err)
+			}
+		}
+	})
+	return epc
+}
+
+func downgradeEnable(t *testing.T, epc *etcdProcessCluster) {
+	t.Log("etcdctl downgrade...")
+	c, err := clientv3.New(clientv3.Config{
+		Endpoints: epc.EndpointsV3(),
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer c.Close()
+	ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+	// TODO: Fix request always timing out even thou it succeeds
+	c.Downgrade(ctx, 1, "3.5.0")
+	cancel()
+
+	expectLog(t, epc, "The server is ready to downgrade")
+}
+
+func stopEtcd(t *testing.T, epc *etcdProcessCluster) {
+	t.Log("Stopping the server...")
+	if err := epc.procs[0].Stop(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func validateVersion(t *testing.T, epc *etcdProcessCluster, expect version.Versions) {
+	t.Log("Validate version")
+	// Two separate calls to expect as it doesn't support multiple matches on the same line
+	executeWithTimeout(t, 20*time.Second, func() {
+		if expect.Server != "" {
+			err := spawnWithExpects(cURLPrefixArgs(epc, "GET", cURLReq{endpoint: "/version"}), nil, `"etcdserver":"`+expect.Server)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+		if expect.Cluster != "" {
+			err := spawnWithExpects(cURLPrefixArgs(epc, "GET", cURLReq{endpoint: "/version"}), nil, `"etcdcluster":"`+expect.Cluster)
+			if err != nil {
+				t.Fatal(err)
+			}
+		}
+	})
+}
+
+func expectLog(t *testing.T, epc *etcdProcessCluster, expectLog string) {
+	t.Helper()
+	executeWithTimeout(t, 30*time.Second, func() {
+		_, err := epc.procs[0].Logs().Expect(expectLog)
+		if err != nil {
+			t.Fatal(err)
+		}
+	})
+}