Merge branch 'add-DR-test' of https://github.com/xiaojingchen/tidb-op…

…erator into add-DR-test

deploy/aliyun/README-CN.md

@@ -62,7 +62,7 @@ monitor_endpoint = 1.2.3.4:3000
 region = cn-hangzhou
 tidb_port = 4000
 tidb_slb_ip = 192.168.5.53
-tidb_version = v2.1.0
+tidb_version = v3.0.0-rc.1
 vpc_id = vpc-bp16wcbu0xhbg833fymmc
 worker_key_file = /root/tidb-operator/deploy/alicloud/credentials/tidb-cluster-node-key.pem
 ```

deploy/aliyun/README.md

@@ -13,11 +13,11 @@
 
 > You can use the Alibaba [Cloud Shell](https://shell.aliyun.com) service, which has all the tools pre-installed and properly configured.
 
-## Overview 
+## Overview
 
 The default setup will create:
- 
-- A new VPC 
+
+- A new VPC
 - An ECS instance as bastion machine
 - A managed ACK(Alibaba Cloud Kubernetes) cluster with the following ECS instance worker nodes:
   - An auto-scaling group of 2 * instances(2c2g) as ACK mandatory workers for system service like CoreDNS
@@ -70,12 +70,12 @@ monitor_endpoint = 1.2.3.4:3000
 region = cn-hangzhou
 tidb_port = 4000
 tidb_slb_ip = 192.168.5.53
-tidb_version = v2.1.0
+tidb_version = v3.0.0-rc.1
 vpc_id = vpc-bp16wcbu0xhbg833fymmc
 worker_key_file = /root/tidb-operator/deploy/alicloud/credentials/tidb-cluster-node-key.pem
 ```
 
-You can then interact with the ACK cluster using `kubectl` and `helm` (`cluster_name` is `tidb-cluster` by default): 
+You can then interact with the ACK cluster using `kubectl` and `helm` (`cluster_name` is `tidb-cluster` by default):
 
 ```shell
 $ export KUBECONFIG=$PWD/credentials/kubeconfig_<cluster_name>
@@ -92,7 +92,7 @@ $ ssh -i credentials/<cluster_name>-bastion-key.pem root@<bastion_ip>
 $ mysql -h <tidb_slb_ip> -P <tidb_port> -u root
 ```
 
-## Monitoring 
+## Monitoring
 
 Visit `<monitor_endpoint>` to view the grafana dashboards. You can find this information in the output of installation.
 
@@ -128,7 +128,7 @@ $ terraform destroy
 
 ## Customize
 
-By default, the terraform script will create a new VPC. You can use an existing VPC by setting `vpc_id` to use an existing VPC. Note that kubernetes node will only be created in available zones that has vswitch existed when using existing VPC. 
+By default, the terraform script will create a new VPC. You can use an existing VPC by setting `vpc_id` to use an existing VPC. Note that kubernetes node will only be created in available zones that has vswitch existed when using existing VPC.
 
 An ecs instance is also created by default as bastion machine to connect to the created TiDB cluster, because the TiDB service is only exposed to intranet. The bastion instance has mysql-cli and sysbench installed that helps you use and test TiDB.
 
@@ -148,4 +148,3 @@ For more customization options, please refer to `variables.tf`
 ## Limitations
 
 You cannot change pod cidr, service cidr and worker instance types once the cluster created.
-

deploy/aliyun/variables.tf

@@ -5,7 +5,7 @@ variable "cluster_name" {
 
 variable "tidb_version" {
   description = "TiDB cluster version"
-  default     = "v2.1.8"
+  default     = "v3.0.0-rc.1"
 }
 
 variable "pd_count" {
@@ -153,4 +153,4 @@ variable "k8s_service_cidr" {
 variable "vpc_cidr" {
   description = "VPC cidr_block, options: [192.168.0.0.0/16, 172.16.0.0/16, 10.0.0.0/8], cannot collidate with kubernetes service cidr and pod cidr. Cannot change once the vpc created."
   default = "192.168.0.0/16"
-}
+}

deploy/aws/README.md

@@ -118,12 +118,12 @@ The initial Grafana login credentials are:
 
 To upgrade the TiDB cluster, edit the `variables.tf` file with your preferred text editor and modify the `tidb_version` variable to a higher version, and then run `terraform apply`.
 
-For example, to upgrade the cluster to version 2.1.10, modify the `tidb_version` to `v2.1.10`:
+For example, to upgrade the cluster to version 3.0.0-rc.1, modify the `tidb_version` to `v3.0.0-rc.2`:
 
 ```
  variable "tidb_version" {
    description = "tidb cluster version"
-   default = "v2.1.10"
+   default = "v3.0.0-rc.2"
  }
 ```
 

deploy/aws/variables.tf

@@ -69,7 +69,7 @@ variable "k8s_version" {
 
 variable "tidb_version" {
   description = "tidb cluster version"
-  default = "v2.1.8"
+  default = "v3.0.0-rc.1"
 }
 
 variable "pd_count" {

deploy/gcp/README.md

@@ -18,7 +18,7 @@ Before deploying, you need to configure several items to guarantee a smooth depl
 
 ### Configure Cloud SDK
 
-After you install Google Cloud SDK, you need to run `gcloud init` to [perform initial setup tasks](https://cloud.google.com/sdk/docs/initializing). 
+After you install Google Cloud SDK, you need to run `gcloud init` to [perform initial setup tasks](https://cloud.google.com/sdk/docs/initializing).
 
 ### Configure APIs
 
@@ -36,16 +36,16 @@ gcloud services enable container.googleapis.com
 
 The terraform script expects three environment variables. You can let Terraform prompt you for them, or `export` them in the `~/.bash_profile` file ahead of time. The required environment variables are:
 
-* `TF_VAR_GCP_CREDENTIALS_PATH`: Path to a valid GCP credentials file. 
-    - It is recommended to create a new service account to be used by Terraform. See [this page](https://cloud.google.com/iam/docs/creating-managing-service-accounts) to create a service account and grant `Project Editor` role to it. 
+* `TF_VAR_GCP_CREDENTIALS_PATH`: Path to a valid GCP credentials file.
+    - It is recommended to create a new service account to be used by Terraform. See [this page](https://cloud.google.com/iam/docs/creating-managing-service-accounts) to create a service account and grant `Project Editor` role to it.
     - See [this page](https://cloud.google.com/iam/docs/creating-managing-service-account-keys) to create service account keys, and choose `JSON` key type during creation. The downloaded `JSON` file that contains the private key is the credentials file you need.
 * `TF_VAR_GCP_REGION`: The region to create the resources in, for example: `us-west1`.
 * `TF_VAR_GCP_PROJECT`: The name of the GCP project.
 
 > *Note*: The service account must have sufficient permissions to create resources in the project. The `Project Editor` primitive will accomplish this.
 
 To set the three environment variables, for example, you can enter in your terminal:
- 
+
 ```bash
 # Replace the values with the path to the JSON file you have downloaded, the GCP region and your GCP project name.
 export TF_VAR_GCP_CREDENTIALS_PATH="/Path/to/my-project.json"
@@ -66,7 +66,7 @@ The default setup creates a new VPC, two subnetworks, and an f1-micro instance a
 
 > *Note*: The number of nodes created depends on how many availability zones there are in the chosen region. Most have 3 zones, but us-central1 has 4. See [Regions and Zones](https://cloud.google.com/compute/docs/regions-zones/) for more information and see the [Customize](#customize) section on how to customize node pools in a regional cluster.
 
-The default setup, as listed above, requires at least 91 CPUs which exceed the default CPU quota of a GCP project. To increase your project's quota, follow the instructions [here](https://cloud.google.com/compute/quotas). You need more CPUs if you need to scale out. 
+The default setup, as listed above, requires at least 91 CPUs which exceed the default CPU quota of a GCP project. To increase your project's quota, follow the instructions [here](https://cloud.google.com/compute/quotas). You need more CPUs if you need to scale out.
 
 Now that you have configured everything needed, you can launch the script to deploy the TiDB cluster:
 
@@ -96,9 +96,9 @@ monitor_port = 3000
 region = us-west1
 tidb_ilb_ip = 172.31.252.20
 tidb_port = 4000
-tidb_version = v2.1.8
+tidb_version = v3.0.0-rc.1
 ```
- 
+
 ## Access the database
 
 After `terraform apply` is successful, the TiDB cluster can be accessed by SSHing into the bastion machine and connecting via MySQL:
@@ -130,12 +130,12 @@ helm ls
 
 To upgrade the TiDB cluster, modify the `tidb_version` variable to a higher version in the `variables.tf` file, and run `terraform apply`.
 
-For example, to upgrade the cluster to the 2.1.10 version, modify the `tidb_version` to `v2.1.10`:
+For example, to upgrade the cluster to the 3.0.0-rc.2 version, modify the `tidb_version` to `v3.0.0-rc.2`:
 
 ```
 variable "tidb_version" {
   description = "TiDB version"
-  default     = "v2.1.10"
+  default     = "v3.0.0-rc.2"
 }
 ```
 
@@ -144,11 +144,11 @@ The upgrading does not finish immediately. You can run `kubectl --kubeconfig cre
 ```sql
 MySQL [(none)]> select tidb_version();
 *************************** 1. row ***************************
-tidb_version(): Release Version: 2.1.10
-Git Commit Hash: v2.1.10
-Git Branch: master
-UTC Build Time: 2019-05-22 11:12:14
-GoVersion: go version go1.12.4 linux/amd64
+tidb_version(): Release Version: v3.0.0-rc.2
+Git Commit Hash: 06f3f63d5a87e7f0436c0618cf524fea7172eb93
+Git Branch: HEAD
+UTC Build Time: 2019-05-28 12:48:52
+GoVersion: go version go1.12 linux/amd64
 Race Enabled: false
 TiKV Min Version: 2.1.0-alpha.1-ff3dd160846b7d1aed9079c389fc188f7f5ea13e
 Check Table Before Drop: false

deploy/gcp/variables.tf

@@ -5,7 +5,7 @@ variable "cluster_name" {
 
 variable "tidb_version" {
   description = "TiDB version"
-  default     = "v2.1.8"
+  default     = "v3.0.0-rc.1"
 }
 
 variable "tidb_operator_version" {

docs/aws-eks-tutorial.md

@@ -45,7 +45,7 @@ For simplicity you can just assign `AdministratorAccess` to the group this user
 belongs to. With more detailed permissions, you will have to be sure you also have
 `AmazonEKSClusterPolicy` and `AmazonEKSServicePolicy` for this user.
 
-Then generate a pair of access keys and keep them safe locally. 
+Then generate a pair of access keys and keep them safe locally.
 
 ## A bit more about Terraform
 
@@ -138,7 +138,7 @@ monitor_endpoint = http://a37987df9710211e9b48c0ae40bc8d7b-1847612729.us-east-2.
 region = us-east-2
 tidb_dns = internal-a37a17c22710211e9b48c0ae40bc8d7b-1891023212.us-east-2.elb.amazonaws.com
 tidb_port = 4000
-tidb_version = v2.1.8
+tidb_version = v3.0.0-rc.1
 ```
 
 > *NOTE*: Be careful about changing instance types for PD and TiKV worker groups as they rely on local SSD. Doing so may break user-data and local volume setup.
@@ -154,7 +154,7 @@ mysql -h <tidb_dns> -P <tidb_port> -u root
 
 It is possible to interact with the cluster via `kubectl` and `helm` with the kubeconfig file that is created `credentials/kubeconfig_aws_tutorial`.
 
-```sh 
+```sh
 # By specifying --kubeconfig
 kubectl --kubeconfig credentials/kubeconfig_aws_tutorial get po -n tidb
 helm --kubeconfig credentials/kubeconfig_aws_tutorial ls

docs/cli-manual.md

@@ -42,10 +42,10 @@ $ mv tkctl /usr/local/bin/tkctl
 BASH
 ```shell
 # setup autocomplete in bash into the current shell, bash-completion package should be installed first.
-source <(tkctl completion bash) 
+source <(tkctl completion bash)
 
 # add autocomplete permanently to your bash shell.
-echo "if hash tkctl 2>/dev/null; then source <(tkctl completion bash); fi" >> ~/.bashrc 
+echo "if hash tkctl 2>/dev/null; then source <(tkctl completion bash); fi" >> ~/.bashrc
 ```
 
 ZSH
@@ -54,7 +54,7 @@ ZSH
 source <(tkctl completion zsh)
 
 # add autocomplete permanently to your zsh shell
-echo "if hash tkctl 2>/dev/null; then source <(tkctl completion zsh); fi" >> ~/.zshrc 
+echo "if hash tkctl 2>/dev/null; then source <(tkctl completion zsh); fi" >> ~/.zshrc
 ```
 
 ## Kubernetes Configuration
@@ -128,9 +128,9 @@ CreationTimestamp:  2019-04-17 17:33:41 +0800 CST
 Overview:
          Phase    Ready  Desired  CPU    Memory  Storage  Version
          -----    -----  -------  ---    ------  -------  -------
-  PD:    Normal   3      3        200m   1Gi     1Gi      pingcap/pd:v2.1.4
-  TiKV:  Normal   3      3        1000m  2Gi     10Gi     pingcap/tikv:v2.1.4
-  TiDB   Upgrade  1      2        500m   1Gi              pingcap/tidb:v2.1.4
+  PD:    Normal   3      3        200m   1Gi     1Gi      pingcap/pd:v3.0.0-rc.1
+  TiKV:  Normal   3      3        1000m  2Gi     10Gi     pingcap/tikv:v3.0.0-rc.1
+  TiDB   Upgrade  1      2        500m   1Gi              pingcap/tidb:v3.0.0-rc.1
 Endpoints(NodePort):
   - 172.16.4.158:31441
   - 172.16.4.155:31441
@@ -252,4 +252,3 @@ These options are mainly used to talk with the kubernetes cluster, there are two
 
 - `--context`: choose the kubernetes cluster
 - `--namespace`: choose the kubernetes namespace
-

docs/local-dind-tutorial.md

@@ -30,7 +30,7 @@ Before deploying a TiDB cluster to Kubernetes, make sure the following requireme
 - Supported filesystem
 
     For Linux users, if the host machine uses an XFS filesystem (default in CentOS 7), it must be formatted with `ftype=1` to enable `d_type` support, see [Docker's documentation](https://docs.docker.com/storage/storagedriver/overlayfs-driver/) for details.
-    
+
     You can check if your filesystem supports `d_type` using command `xfs_info / | grep ftype`, where `/` is the data directory of you installed Docker daemon.
 
     If your root directory `/` uses XFS without `d_type` support, but there is another partition does, or is using another filesystem, it is also possible to change the data directory of Docker to use that partition.
@@ -50,7 +50,7 @@ Before deploying a TiDB cluster to Kubernetes, make sure the following requireme
     # Overrite config
     cat << EOF > /etc/systemd/system/docker.service.d/docker-storage.conf
     [Service]
-    ExecStart= 
+    ExecStart=
     ExecStart=/usr/bin/dockerd --data-root /data/docker -H fd:// --containerd=/run/containerd/containerd.sock
     EOF
 
@@ -138,8 +138,8 @@ $ kubectl get pods --namespace tidb -l app.kubernetes.io/instance=demo -o wide -
 
 # Get basic information of the TiDB cluster
 $ kubectl get tidbcluster -n tidb
-NAME   PD                  STORAGE   READY   DESIRE   TIKV                  STORAGE   READY   DESIRE   TIDB                  READY   DESIRE
-demo   pingcap/pd:v2.1.8   1Gi       3       3        pingcap/tikv:v2.1.8   10Gi      3       3        pingcap/tidb:v2.1.8   2       2
+NAME   PD                       STORAGE   READY   DESIRE   TIKV                       STORAGE   READY   DESIRE   TIDB                       READY   DESIRE
+demo   pingcap/pd:v3.0.0-rc.1   1Gi       3       3        pingcap/tikv:v3.0.0-rc.1   10Gi      3       3        pingcap/tidb:v3.0.0-rc.1   2       2
 
 $ kubectl get statefulset -n tidb
 NAME        DESIRED   CURRENT   AGE
@@ -286,7 +286,7 @@ Use `kubectl get pod -n tidb` to verify the number of each compoments equal to v
 
 1. Edit the `charts/tidb-cluster/values.yaml` file with your preferred text editor.
 
-    For example, change the version of PD/TiKV/TiDB `image` to `v2.1.10`.
+    For example, change the version of PD/TiKV/TiDB `image` to `v3.0.0-rc.2`.
 
 2. Run the following command to apply the changes:
 
@@ -299,11 +299,11 @@ Use `kubectl get pod -n tidb` to verify that all pods are in `Running` state. Th
 ```sh
 MySQL [(none)]> select tidb_version();
 *************************** 1. row ***************************
-tidb_version(): Release Version: 2.1.10
-Git Commit Hash: v2.1.10
-Git Branch: master
-UTC Build Time: 2019-05-22 11:12:14
-GoVersion: go version go1.12.4 linux/amd64
+tidb_version(): Release Version: v3.0.0-rc.2
+Git Commit Hash: 06f3f63d5a87e7f0436c0618cf524fea7172eb93
+Git Branch: HEAD
+UTC Build Time: 2019-05-28 12:48:52
+GoVersion: go version go1.12 linux/amd64
 Race Enabled: false
 TiKV Min Version: 2.1.0-alpha.1-ff3dd160846b7d1aed9079c389fc188f7f5ea13e
 Check Table Before Drop: false

tests/actions.go

@@ -170,7 +170,9 @@ type OperatorActions interface {
 	CheckDataRegionDisasterTolerance(info *TidbClusterConfig) error
 	CheckDataRegionDisasterToleranceOrDie(info *TidbClusterConfig)
 	GetTidbMemberAssignedNodes(info *TidbClusterConfig) (map[string]string, error)
+	GetTidbMemberAssignedNodesOrDie(info *TidbClusterConfig) map[string]string
 	CheckTidbMemberAssignedNodes(info *TidbClusterConfig, oldAssignedNodes map[string]string) error
+	CheckTidbMemberAssignedNodesOrDie(info *TidbClusterConfig, oldAssignedNodes map[string]string)
 }
 
 type operatorActions struct {
@@ -623,7 +625,16 @@ func (oa *operatorActions) GetTidbMemberAssignedNodes(info *TidbClusterConfig) (
 	return assignedNodes, nil
 }
 
+func (oa *operatorActions) GetTidbMemberAssignedNodesOrDie(info *TidbClusterConfig) map[string]string {
+	result, err := oa.GetTidbMemberAssignedNodes(info)
+	if err != nil {
+		slack.NotifyAndPanic(err)
+	}
+	return result
+}
+
 func (oa *operatorActions) CheckTidbMemberAssignedNodes(info *TidbClusterConfig, oldAssignedNodes map[string]string) error {
+	glog.Infof("checking tidb member [%s/%s] assigned nodes", info.Namespace, info.ClusterName)
 	assignedNodes, err := oa.GetTidbMemberAssignedNodes(info)
 	if err != nil {
 		return err
@@ -637,6 +648,12 @@ func (oa *operatorActions) CheckTidbMemberAssignedNodes(info *TidbClusterConfig,
 	return nil
 }
 
+func (oa *operatorActions) CheckTidbMemberAssignedNodesOrDie(info *TidbClusterConfig, oldAssignedNodes map[string]string) {
+	if err := oa.CheckTidbMemberAssignedNodes(info, oldAssignedNodes); err != nil {
+		slack.NotifyAndPanic(err)
+	}
+}
+
 func (oa *operatorActions) CheckTidbClusterStatus(info *TidbClusterConfig) error {
 	glog.Infof("checking tidb cluster [%s/%s] status", info.Namespace, info.ClusterName)
 

tests/cmd/stability/main.go

@@ -47,11 +47,14 @@ func main() {
 	upgardeTiDBVersions := conf.GetUpgradeTidbVersionsOrDie()
 
 	operatorCfg := &tests.OperatorConfig{
-		Namespace:          "pingcap",
-		ReleaseName:        "operator",
-		Image:              conf.OperatorImage,
-		Tag:                conf.OperatorTag,
-		SchedulerImage:     "gcr.io/google-containers/hyperkube",
+		Namespace:      "pingcap",
+		ReleaseName:    "operator",
+		Image:          conf.OperatorImage,
+		Tag:            conf.OperatorTag,
+		SchedulerImage: "gcr.io/google-containers/hyperkube",
+		SchedulerFeatures: []string{
+			"StableScheduling",
+		},
 		LogLevel:           "2",
 		WebhookServiceName: "webhook-service",
 		WebhookSecretName:  "webhook-secret",
@@ -245,12 +248,16 @@ func run(oa tests.OperatorActions,
 
 	// upgrade cluster1 and cluster2
 	firstUpgradeVersion := upgardeTiDBVersions[0]
+	assignedNodes1 := oa.GetTidbMemberAssignedNodesOrDie(cluster1)
+	assignedNodes2 := oa.GetTidbMemberAssignedNodesOrDie(cluster2)
 	cluster1.UpgradeAll(firstUpgradeVersion)
 	cluster2.UpgradeAll(firstUpgradeVersion)
 	oa.UpgradeTidbClusterOrDie(cluster1)
 	oa.UpgradeTidbClusterOrDie(cluster2)
 	oa.CheckTidbClusterStatusOrDie(cluster1)
 	oa.CheckTidbClusterStatusOrDie(cluster2)
+	oa.CheckTidbMemberAssignedNodesOrDie(cluster1, assignedNodes1)
+	oa.CheckTidbMemberAssignedNodesOrDie(cluster2, assignedNodes2)
 
 	// after upgrade cluster, clean webhook
 	oa.CleanWebHookAndService(operatorCfg)