New stability test case for ConfigMap rollout

charts/tidb-cluster/templates/scripts/_start_pd.sh.tpl

@@ -9,6 +9,9 @@
 #
 
 set -uo pipefail
+
+{{ .Values.pd.preStartScript }}
+
 ANNOTATIONS="/etc/podinfo/annotations"
 
 if [[ ! -f "${ANNOTATIONS}" ]]

charts/tidb-cluster/templates/scripts/_start_tidb.sh.tpl

@@ -8,6 +8,9 @@
 #   runmode="normal/debug"
 #
 set -uo pipefail
+
+{{ .Values.tidb.preStartScript }}
+
 ANNOTATIONS="/etc/podinfo/annotations"
 
 if [[ ! -f "${ANNOTATIONS}" ]]

charts/tidb-cluster/templates/scripts/_start_tikv.sh.tpl

@@ -9,6 +9,9 @@
 #
 
 set -uo pipefail
+
+{{ .Values.tikv.preStartScript }}
+
 ANNOTATIONS="/etc/podinfo/annotations"
 
 if [[ ! -f "${ANNOTATIONS}" ]]

tests/actions.go

@@ -151,6 +151,7 @@ type OperatorActions interface {
 	CheckK8sAvailableOrDie(excludeNodes map[string]string, excludePods map[string]*corev1.Pod)
 	CheckOperatorAvailable(operatorConfig *OperatorConfig) error
 	CheckTidbClustersAvailable(infos []*TidbClusterConfig) error
+	CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig)
 	CheckOneEtcdDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
 	CheckOneApiserverDownOrDie(operatorConfig *OperatorConfig, clusters []*TidbClusterConfig, faultNode string)
 	RegisterWebHookAndService(info *OperatorConfig) error
@@ -202,6 +203,7 @@ type OperatorConfig struct {
 	WebhookSecretName  string
 	WebhookConfigName  string
 	Context            *apimachinery.CertContext
+	ImagePullPolicy    corev1.PullPolicy
 }
 
 type TidbClusterConfig struct {
@@ -231,6 +233,10 @@ type TidbClusterConfig struct {
 	TiDBTokenLimit      int
 	PDLogLevel          string
 
+	PDPreStartScript   string
+	TiDBPreStartScript string
+	TiKVPreStartScript string
+
 	BlockWriteConfig blockwriter.Config
 	GrafanaClient    *metrics.Client
 }
@@ -286,6 +292,9 @@ func (tc *TidbClusterConfig) TidbClusterHelmSetString(m map[string]string) strin
 		"tidb.initSql":            tc.InitSQL,
 		"monitor.create":          strconv.FormatBool(tc.Monitor),
 		"enableConfigMapRollout":  strconv.FormatBool(tc.EnableConfigMapRollout),
+		"pd.preStartScript":       tc.PDPreStartScript,
+		"tikv.preStartScript":     tc.TiKVPreStartScript,
+		"tidb.preStartScript":     tc.TiDBPreStartScript,
 	}
 
 	if tc.PDMaxReplicas > 0 {
@@ -327,6 +336,7 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string {
 		"scheduler.logLevel":               "2",
 		"controllerManager.replicas":       "2",
 		"scheduler.replicas":               "2",
+		"imagePullPolicy":                  string(oi.ImagePullPolicy),
 	}
 	if oi.SchedulerTag != "" {
 		set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag
@@ -1583,7 +1593,7 @@ func notFound(res string) bool {
 }
 
 func (oa *operatorActions) cloneOperatorRepo() error {
-	cmd := fmt.Sprintf("git clone https://github.com/pingcap/tidb-operator.git %s", oa.cfg.OperatorRepoDir)
+	cmd := fmt.Sprintf("git clone %s %s", oa.cfg.OperatorRepoUrl, oa.cfg.OperatorRepoDir)
 	glog.Info(cmd)
 	res, err := exec.Command("/bin/sh", "-c", cmd).CombinedOutput()
 	if err != nil && !strings.Contains(string(res), "already exists") {

tests/cmd/e2e/main.go

@@ -15,6 +15,7 @@ package main
 
 import (
 	"fmt"
+	"k8s.io/api/core/v1"
 	_ "net/http/pprof"
 	"time"
 
@@ -51,6 +52,7 @@ func main() {
 		WebhookServiceName: "webhook-service",
 		WebhookSecretName:  "webhook-secret",
 		WebhookConfigName:  "webhook-config",
+		ImagePullPolicy:    v1.PullIfNotPresent,
 	}
 
 	// start a http server in goruntine

tests/cmd/stability/main.go

@@ -15,8 +15,10 @@ package main
 
 import (
 	"fmt"
+	"k8s.io/api/core/v1"
 	"net/http"
 	_ "net/http/pprof"
+	"strconv"
 	"time"
 
 	"github.com/pingcap/tidb-operator/tests/slack"
@@ -55,6 +57,7 @@ func main() {
 		WebhookServiceName: "webhook-service",
 		WebhookSecretName:  "webhook-secret",
 		WebhookConfigName:  "webhook-config",
+		ImagePullPolicy:    v1.PullAlways,
 	}
 
 	// TODO remove this
@@ -98,8 +101,13 @@ func main() {
 			"binlog.drainer.workerCount": "1024",
 			"binlog.drainer.txnBatch":    "512",
 		},
-		Monitor:          true,
-		BlockWriteConfig: conf.BlockWriter,
+		Monitor:                true,
+		BlockWriteConfig:       conf.BlockWriter,
+		PDMaxReplicas:          3,
+		TiKVGrpcConcurrency:    4,
+		TiDBTokenLimit:         1000,
+		PDLogLevel:             "info",
+		EnableConfigMapRollout: true,
 	}
 	cluster2 := &tests.TidbClusterConfig{
 		Namespace:        clusterName2,
@@ -132,9 +140,14 @@ func main() {
 			"monitor.persistent": "true",
 			"discovery.image":    conf.OperatorImage,
 		},
-		Args:             map[string]string{},
-		Monitor:          true,
-		BlockWriteConfig: conf.BlockWriter,
+		Args:                   map[string]string{},
+		Monitor:                true,
+		BlockWriteConfig:       conf.BlockWriter,
+		PDMaxReplicas:          3,
+		TiKVGrpcConcurrency:    4,
+		TiDBTokenLimit:         1000,
+		PDLogLevel:             "info",
+		EnableConfigMapRollout: false,
 	}
 
 	// cluster backup and restore
@@ -201,6 +214,34 @@ func main() {
 	oa.CheckTidbClusterStatusOrDie(cluster1)
 	oa.CheckTidbClusterStatusOrDie(cluster2)
 
+	// cluster1: bad configuration change case
+	cluster1.TiDBPreStartScript = strconv.Quote("exit 1")
+	oa.UpgradeTidbClusterOrDie(cluster1)
+	cluster1.TiKVPreStartScript = strconv.Quote("exit 1")
+	oa.UpgradeTidbClusterOrDie(cluster1)
+	cluster1.PDPreStartScript = strconv.Quote("exit 1")
+	oa.UpgradeTidbClusterOrDie(cluster1)
+
+	time.Sleep(30 * time.Second)
+	oa.CheckTidbClustersAvailableOrDie([]*tests.TidbClusterConfig{cluster1})
+
+	// rollback cluster1
+	cluster1.PDPreStartScript = strconv.Quote("")
+	cluster1.TiKVPreStartScript = strconv.Quote("")
+	cluster1.TiDBPreStartScript = strconv.Quote("")
+	oa.UpgradeTidbClusterOrDie(cluster1)
+	oa.CheckTidbClusterStatusOrDie(cluster1)
+
+	// cluster2: enable and normal configuration change case
+	cluster2.EnableConfigMapRollout = true
+	oa.UpgradeTidbClusterOrDie(cluster2)
+	oa.CheckTidbClusterStatusOrDie(cluster2)
+	cluster2.UpdatePdMaxReplicas(conf.PDMaxReplicas).
+		UpdateTiKVGrpcConcurrency(conf.TiKVGrpcConcurrency).
+		UpdateTiDBTokenLimit(conf.TiDBTokenLimit)
+	oa.UpgradeTidbClusterOrDie(cluster2)
+	oa.CheckTidbClusterStatusOrDie(cluster2)
+
 	// after upgrade cluster, clean webhook
 	oa.CleanWebHookAndService(operatorCfg)
 

tests/config.go

@@ -45,6 +45,7 @@ type Config struct {
 	BlockWriter blockwriter.Config `yaml:"block_writer,omitempty"`
 
 	// For local test
+	OperatorRepoUrl string `yaml:"operator_repo_url" json:"operator_repo_url"`
 	OperatorRepoDir string `yaml:"operator_repo_dir" json:"operator_repo_dir"`
 	// chart dir
 	ChartDir string `yaml:"chart_dir" json:"chart_dir"`
@@ -61,6 +62,7 @@ type Nodes struct {
 // NewConfig creates a new config.
 func NewConfig() (*Config, error) {
 	cfg := &Config{
+		OperatorRepoUrl: "https://github.com/pingcap/tidb-operator.git",
 
 		PDMaxReplicas:       5,
 		TiDBTokenLimit:      1024,

tests/failover.go

@@ -584,6 +584,12 @@ func (oa *operatorActions) CheckTidbClustersAvailable(infos []*TidbClusterConfig
 
 }
 
+func (oa *operatorActions) CheckTidbClustersAvailableOrDie(infos []*TidbClusterConfig) {
+	if err := oa.CheckTidbClustersAvailable(infos); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 var testTableName = "testTable"
 
 func (oa *operatorActions) addDataToCluster(info *TidbClusterConfig) (bool, error) {