Skip to content

Commit

Permalink
Add auto-scaling e2e test (#2123) (#2142)
Browse files Browse the repository at this point in the history 
* add auto-scaling e2e
  • Loading branch information
@sre-bot
sre-bot authored Apr 9, 2020
1 parent c195f7a commit 159cba5
Show file tree
Hide file tree
Showing 2 changed files with 316 additions and 0 deletions.
295 changes: 295 additions & 0 deletions tests/e2e/tidbcluster/serial.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ import (
"encoding/json"
"fmt"
_ "net/http/pprof"
"strconv"
"time"

"github.com/onsi/ginkgo"
Expand All @@ -29,11 +30,13 @@ import (
"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/scheme"
"github.com/pingcap/tidb-operator/pkg/util"
"github.com/pingcap/tidb-operator/tests"
e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image"
utilpod "github.com/pingcap/tidb-operator/tests/e2e/util/pod"
"github.com/pingcap/tidb-operator/tests/e2e/util/portforward"
"github.com/pingcap/tidb-operator/tests/e2e/util/proxiedpdclient"
utilstatefulset "github.com/pingcap/tidb-operator/tests/e2e/util/statefulset"
"github.com/pingcap/tidb-operator/tests/pkg/fixture"
v1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -713,4 +716,296 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() {
})
})

ginkgo.Context("[Feature: AutoScaling]", func() {
var ocfg *tests.OperatorConfig
var oa tests.OperatorActions

ginkgo.BeforeEach(func() {
ocfg = &tests.OperatorConfig{
Namespace: ns,
ReleaseName: "operator",
Image: cfg.OperatorImage,
Tag: cfg.OperatorTag,
LogLevel: "4",
TestMode: true,
WebhookEnabled: true,
PodWebhookEnabled: true,
Features: []string{
"AutoScaling=true",
},
}
oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f)
ginkgo.By("Installing CRDs")
oa.CleanCRDOrDie()
oa.InstallCRDOrDie(ocfg)
ginkgo.By("Installing tidb-operator")
oa.CleanOperatorOrDie(ocfg)
oa.DeployOperatorOrDie(ocfg)
})

ginkgo.AfterEach(func() {
ginkgo.By("Uninstall tidb-operator")
oa.CleanOperatorOrDie(ocfg)
ginkgo.By("Uninstalling CRDs")
oa.CleanCRDOrDie()
})

ginkgo.It("auto-scaling TidbCluster", func() {
clusterName := "auto-scaling"
tc := fixture.GetTidbCluster(ns, clusterName, utilimage.TiDBV3Version)
tc.Spec.PD.Replicas = 1
tc.Spec.TiKV.Replicas = 3
tc.Spec.TiDB.Replicas = 2
_, err := cli.PingcapV1alpha1().TidbClusters(ns).Create(tc)
framework.ExpectNoError(err, "Create TidbCluster error")
err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second)
framework.ExpectNoError(err, "Check TidbCluster error")
monitor := fixture.NewTidbMonitor("monitor", ns, tc, false, false)
_, err = cli.PingcapV1alpha1().TidbMonitors(ns).Create(monitor)
framework.ExpectNoError(err, "Create TidbMonitor error")
err = tests.CheckTidbMonitor(monitor, c, fw)
framework.ExpectNoError(err, "Check TidbMonitor error")
tac := fixture.GetTidbClusterAutoScaler("auto-scaler", ns, tc, monitor)

//TODO we should mock the tidbmonitor metrics data to check the metrics calculating
// Currently these steps are checked by unit test
// For now, we make minReplicas and maxReplicas equal to run the auto-scaling

// Scale Tikv To 4 replicas and Check
tac.Spec.TiKV = &v1alpha1.TikvAutoScalerSpec{
BasicAutoScalerSpec: v1alpha1.BasicAutoScalerSpec{
MaxReplicas: 4,
MinReplicas: pointer.Int32Ptr(4),
},
}
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Create(tac)
framework.ExpectNoError(err, "Create TidbMonitorClusterAutoScaler error")
pdClient, cancel, err := proxiedpdclient.NewProxiedPDClient(c, fw, ns, clusterName, false, nil)
framework.ExpectNoError(err, "create pdapi error")
defer cancel()
var firstScaleTimestamp int64
err = wait.Poll(10*time.Second, 10*time.Minute, func() (done bool, err error) {
tc, err := cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
// check replicas
if tc.Spec.TiKV.Replicas != int32(4) {
klog.Infof("tikv haven't auto-scale to 4 replicas")
return false, nil
}
if len(tc.Status.TiKV.Stores) != 4 {
klog.Infof("tikv's stores haven't auto-scale to 4")
return false, nil
}
// check annotations
if tc.Annotations == nil || len(tc.Annotations) < 1 {
klog.Infof("tc haven't marked any annotation")
return false, nil
}
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tac.Annotations == nil || len(tac.Annotations) < 1 {
klog.Infof("tac haven't marked any annotation")
return false, nil
}
v, ok := tac.Annotations[label.AnnTiKVLastAutoScalingTimestamp]
if !ok {
klog.Infof("tac haven't marked any annotation")
return false, nil
}
firstScaleTimestamp, err = strconv.ParseInt(v, 10, 64)
if err != nil {
return false, err
}
// check store label
storeId := ""
for k, v := range tc.Status.TiKV.Stores {
if v.PodName == util.GetPodName(tc, v1alpha1.TiKVMemberType, int32(3)) {
storeId = k
break
}
}
if storeId == "" {
return false, nil
}
sid, err := strconv.ParseUint(storeId, 10, 64)
if err != nil {
return false, err
}
info, err := pdClient.GetStore(sid)
if err != nil {
return false, nil
}
for _, label := range info.Store.Labels {
if label.Key == "specialUse" && label.Value == "hotRegion" {
return true, nil
}
}
klog.Infof("tikv auto-scale out haven't find the special label")
return false, nil
})
framework.ExpectNoError(err, "check tikv auto-scale to 4 error")
klog.Info("success to check tikv auto scale-out to 4 replicas")

// Scale Tikv To 3 replicas and Check
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "Get TidbCluster AutoScaler err")
tac.Spec.TiKV = &v1alpha1.TikvAutoScalerSpec{
BasicAutoScalerSpec: v1alpha1.BasicAutoScalerSpec{
MaxReplicas: 3,
MinReplicas: pointer.Int32Ptr(3),
ScaleInIntervalSeconds: pointer.Int32Ptr(100),
},
}
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Update(tac)
framework.ExpectNoError(err, "Update TidbMonitorClusterAutoScaler error")
err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
tc, err = cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tc.Spec.TiKV.Replicas != 3 {
klog.Info("tikv haven't auto-scale to 3 replicas")
return false, nil
}
if len(tc.Status.TiKV.Stores) != 3 {
klog.Info("tikv's store haven't auto-scale to 3")
return false, nil
}
if tc.Annotations != nil && len(tc.Annotations) > 0 {
_, ok := tc.Annotations[label.AnnTiKVAutoScalingOutOrdinals]
if ok {
klog.Infof("tikv auto-scale out annotation still exists")
return false, nil
}
}
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tac.Annotations == nil || len(tac.Annotations) < 1 {
klog.Infof("tc haven't marked any annotation")
return false, nil
}
v, ok := tac.Annotations[label.AnnTiKVLastAutoScalingTimestamp]
if !ok {
klog.Infof("tac haven't marked any annotation")
return false, nil
}
secondTs, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return false, err
}
if secondTs == firstScaleTimestamp {
klog.Info("tikv haven't scale yet")
return false, nil
}
if secondTs-firstScaleTimestamp < 100 {
return false, fmt.Errorf("tikv second scale's interval isn't meeting the interval requirement")
}
return true, nil
})
framework.ExpectNoError(err, "check tikv auto-scale to 3 error")
klog.Info("success to check tikv auto scale-in to 3 replicas")

// Scale Tidb to 3 replicas and Check
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "Get TidbCluster AutoScaler err")
tac.Spec.TiKV = nil
tac.Spec.TiDB = &v1alpha1.TidbAutoScalerSpec{
BasicAutoScalerSpec: v1alpha1.BasicAutoScalerSpec{
MaxReplicas: 3,
MinReplicas: pointer.Int32Ptr(3),
},
}
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Update(tac)
framework.ExpectNoError(err, "Update TidbMonitorClusterAutoScaler error")

err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
tc, err = cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tc.Spec.TiDB.Replicas != 3 {
klog.Info("tidb haven't auto-scaler to 3 replicas")
return false, nil
}
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tac.Annotations == nil || len(tac.Annotations) < 1 {
klog.Info("tac haven't marked any annotations")
return false, nil
}
v, ok := tac.Annotations[label.AnnTiDBLastAutoScalingTimestamp]
if !ok {
klog.Info("tac haven't marked tidb auto-scaler timstamp annotation")
return false, nil
}
firstScaleTimestamp, err = strconv.ParseInt(v, 10, 64)
if err != nil {
return false, err
}
return true, nil
})
framework.ExpectNoError(err, "check tidb auto-scale to 3 error")
klog.Infof("success to check tidb auto scale-out to 3 replicas")

// Scale Tidb to 2 Replicas and Check
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
framework.ExpectNoError(err, "Get TidbCluster AutoScaler err")
tac.Spec.TiKV = nil
tac.Spec.TiDB = &v1alpha1.TidbAutoScalerSpec{
BasicAutoScalerSpec: v1alpha1.BasicAutoScalerSpec{
MaxReplicas: 2,
MinReplicas: pointer.Int32Ptr(2),
ScaleInIntervalSeconds: pointer.Int32Ptr(100),
},
}
_, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Update(tac)
framework.ExpectNoError(err, "Update TidbMonitorClusterAutoScaler error")

err = wait.Poll(5*time.Second, 5*time.Minute, func() (done bool, err error) {
tc, err = cli.PingcapV1alpha1().TidbClusters(tc.Namespace).Get(tc.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tc.Spec.TiDB.Replicas != 2 {
klog.Info("tidb haven't auto-scaler to 2 replicas")
return false, nil
}
tac, err = cli.PingcapV1alpha1().TidbClusterAutoScalers(ns).Get(tac.Name, metav1.GetOptions{})
if err != nil {
return false, nil
}
if tac.Annotations == nil || len(tac.Annotations) < 1 {
klog.Info("tac haven't marked any annotations")
return false, nil
}
v, ok := tac.Annotations[label.AnnTiDBLastAutoScalingTimestamp]
if !ok {
klog.Info("tac haven't marked tidb auto-scale timestamp")
return false, nil
}
secondTs, err := strconv.ParseInt(v, 10, 64)
if err != nil {
return false, err
}
if secondTs == firstScaleTimestamp {
klog.Info("tidb haven't scale yet")
return false, nil
}
if secondTs-firstScaleTimestamp < 100 {
return false, fmt.Errorf("tikv second scale's interval isn't meeting the interval requirement")
}
return true, nil
})
framework.ExpectNoError(err, "check tidb auto-scale to 2 error")
klog.Info("success to check auto scale-in tidb to 2 replicas")
})
})
})
21 changes: 21 additions & 0 deletions tests/pkg/fixture/fixture.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -351,3 +351,24 @@ func GetRestoreCRDWithBR(tc *v1alpha1.TidbCluster, backupFolder string) *v1alpha
},
}
}

func GetTidbClusterAutoScaler(name, ns string, tc *v1alpha1.TidbCluster, tm *v1alpha1.TidbMonitor) *v1alpha1.TidbClusterAutoScaler {
return &v1alpha1.TidbClusterAutoScaler{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Namespace: ns,
},
Spec: v1alpha1.TidbClusterAutoScalerSpec{
Cluster: v1alpha1.TidbClusterRef{
Name: tc.Name,
Namespace: tc.Namespace,
},
Monitor: &v1alpha1.TidbMonitorRef{
Name: tm.Name,
Namespace: tm.Namespace,
},
TiKV: nil,
TiDB: nil,
},
}
}

0 comments on commit 159cba5

Please sign in to comment.