helm deployment

pingcap · Nov 6, 2019 · 4fcb16e · 4fcb16e
1 parent 8fecc00
commit 4fcb16e
Show file tree

Hide file tree

Showing 12 changed files with 173 additions and 75 deletions.
diff --git a/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml
@@ -1,24 +1,33 @@
+{{- if .Values.advancedStatefulset.create }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: advanced-statefulset-controller
-  namespace: pingcap
   labels:
-    app: advanced-statefulset-controller
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 spec:
-  replicas: 1
+  replicas: {{ .Values.advancedStatefulset.replicas }}
   selector:
     matchLabels:
-      app: advanced-statefulset-controller
+      app.kubernetes.io/name: {{ template "chart.name" . }}
+      app.kubernetes.io/instance: {{ .Release.Name }}
+      app.kubernetes.io/component: advanced-statefulset-controller
   template:
     metadata:
       labels:
-        app: advanced-statefulset-controller
+        app.kubernetes.io/name: {{ template "chart.name" . }}
+        app.kubernetes.io/instance: {{ .Release.Name }}
+        app.kubernetes.io/component: advanced-statefulset-controller
     spec:
+      serviceAccountName: advanced-statefulset-controller
       containers:
       - name: advanced-statefulset-controller
-        image: quay.io/cofyc/advanced-statefulset:latest
-        imagePullPolicy: Always
+        image: {{ .Values.advancedStatefulset.image }}
+        imagePullPolicy: {{ .Values.advancedStatefulset.imagePullPolicy | default "IfNotPresent" }}
         args:
         - --v=4
         - --leader-elect
@@ -29,4 +38,18 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-      serviceAccountName: advanced-statefulset-controller
+        resources:
+{{ toYaml .Values.advancedStatefulset.resources | indent 12 }}
+    {{- with .Values.advancedStatefulset.nodeSelector }}
+      nodeSelector:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.advancedStatefulset.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+    {{- with .Values.advancedStatefulset.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+    {{- end }}
+{{- end }}
diff --git a/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml
@@ -1,13 +1,25 @@
+{{- if .Values.advancedStatefulset.create }}
 apiVersion: v1
 kind: ServiceAccount
 metadata:
   name: advanced-statefulset-controller
-  namespace: pingcap
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
   name: advanced-statefulset-controller
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 rules:
 - apiGroups:
   - apps.pingcap.com
@@ -34,20 +46,32 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRoleBinding
 metadata:
   name: advanced-statefulset-controller
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: ClusterRole
   name: advanced-statefulset-controller
 subjects:
 - kind: ServiceAccount
   name: advanced-statefulset-controller
-  namespace: pingcap
+  namespace: {{ .Release.Namespace }}
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: Role
 metadata:
   name: advanced-statefulset-controller
-  namespace: pingcap
+  namespace: {{ .Release.Namespace }}
+  labels:
+    app.kubernetes.io/name: {{ template "chart.name" . }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/component: advanced-statefulset-controller
+    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+"  "_" }}
 rules:
 - apiGroups:
   - ''
@@ -60,12 +84,13 @@ apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
 metadata:
   name: advanced-statefulset-controller
-  namespace: pingcap
+  namespace: {{ .Release.Namespace }}
 roleRef:
   apiGroup: rbac.authorization.k8s.io
   kind: Role
   name: advanced-statefulset-controller
 subjects:
 - kind: ServiceAccount
   name: advanced-statefulset-controller
-  namespace: pingcap
+  namespace: {{ .Release.Namespace }}
+{{- end }}
diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml
@@ -49,8 +49,8 @@ spec:
           {{- if .Values.testMode }}
           - -test-mode={{ .Values.testMode }}
           {{- end}}
-          {{- if .Values.controllerManager.features }}
-          - -features={{ join "," .Values.controllerManager.features }}
+          {{- if .Values.features }}
+          - -features={{ join "," .Values.features }}
           {{- end }}
         env:
           - name: NAMESPACE

diff --git a/charts/tidb-operator/templates/scheduler-deployment.yaml b/charts/tidb-operator/templates/scheduler-deployment.yaml
@@ -35,8 +35,8 @@ spec:
           - /usr/local/bin/tidb-scheduler
           - -v={{ .Values.scheduler.logLevel }}
           - -port=10262
-        {{- if .Values.scheduler.features }}
-          - -features={{ join "," .Values.scheduler.features }}
+        {{- if .Values.features }}
+          - -features={{ join "," .Values.features }}
         {{- end }}
       - name: kube-scheduler
         image: {{ required "scheduler.kubeSchedulerImageName is required" .Values.scheduler.kubeSchedulerImageName }}:{{ .Values.scheduler.kubeSchedulerImageTag | default (split "-" .Capabilities.KubeVersion.GitVersion)._0 }}

diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml
@@ -18,6 +18,23 @@ defaultStorageClassName: local-storage
 # tidbBackupManagerImage: pingcap/tidb-backup-manager:latest
 # defaultBackupStorageClassName: local-storage
 
+#
+# Enable or disable tidb-operator features:
+#
+#   StableScheduling (default: true)
+#     Enable stable scheduling of tidb servers.
+#
+#   AdvancedStatefulSet (default: false)
+#     If enabled, tidb-operator will use AdvancedStatefulSet to manage pods
+#     instead of Kubernetes StatefulSet.
+#     It's ok to turn it on if this feature is not enabled. However it's not ok
+#     to turn it off when the tidb-operator already uses AdvancedStatefulSet to
+#     manage pods. This is in alpha phrase.
+#
+features: {}
+# - AdvancedStatefulSet=false
+# - StableScheduling=true
+
 controllerManager:
   # With rbac.create=false, the user is responsible for creating this account
   # With rbac.create=true, this service account will be created
@@ -54,8 +71,6 @@ controllerManager:
   #   operator: Equal
   #   value: tidb-operator
   #   effect: "NoSchedule"
-  # features:
-  # - AdvancedStatefulSet=true
 
 scheduler:
   # With rbac.create=false, the user is responsible for creating this account
@@ -65,8 +80,6 @@ scheduler:
   logLevel: 2
   replicas: 1
   schedulerName: tidb-scheduler
-  # features:
-  # - StableScheduling=true
   resources:
     limits:
       cpu: 250m
@@ -126,3 +139,36 @@ apiserver:
   #   value: tidb-operator
   #   effect: "NoSchedule"
 
+# When AdvancedStatefulSet feature is enabled, you must install
+# AdvancedStatefulSet controller.
+# Note that AdvancedStatefulSet CRD must be installed manually via the following
+# command:
+#   kubectl apply -f manifests/advanced-statefulset-crd.v1beta1.yaml
+advancedStatefulset:
+  create: false
+  image: quay.io/cofyc/advanced-statefulset:latest
+  imagePullPolicy: IfNotPresent
+  replicas: 1
+  resources:
+    limits:
+      cpu: 500m
+      memory: 300Mi
+    requests:
+      cpu: 200m
+      memory: 50Mi
+  # This will default to matching your kubernetes version
+  # kubeSchedulerImageTag:
+  ## affinity defines pod scheduling rules,affinity default settings is empty.
+  ## please read the affinity document before set your scheduling rule:
+  ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity
+  affinity: {}
+  ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels
+  ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector
+  nodeSelector: {}
+  ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints.
+  ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration
+  tolerations: []
+  # - key: node-role
+  #   operator: Equal
+  #   value: tidb-operator
+  #   effect: "NoSchedule"
diff --git a/.../tidb-operator/templates/crd.v1beta1.yaml → ...sts/advanced-statefulset-crd.v1beta1.yaml b/.../tidb-operator/templates/crd.v1beta1.yaml → ...sts/advanced-statefulset-crd.v1beta1.yaml
diff --git a/tests/actions.go b/tests/actions.go
@@ -32,9 +32,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/ghodss/yaml"
 	// To register MySQL driver
 	_ "github.com/go-sql-driver/mysql"
+
+	"github.com/cofyc/advanced-statefulset/pkg/apis/apps/v1alpha1/helper"
+	asclientset "github.com/cofyc/advanced-statefulset/pkg/client/clientset/versioned"
+	"github.com/ghodss/yaml"
 	pingcapErrors "github.com/pingcap/errors"
 	"github.com/pingcap/kvproto/pkg/metapb"
 	"github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1"
@@ -58,6 +61,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
+	typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1"
 	glog "k8s.io/klog"
 )
 
@@ -76,12 +80,15 @@ const (
 
 func NewOperatorActions(cli versioned.Interface,
 	kubeCli kubernetes.Interface,
+	asCli asclientset.Interface,
 	pollInterval time.Duration,
 	cfg *Config,
 	clusters []*TidbClusterConfig) OperatorActions {
 	oa := &operatorActions{
 		cli:          cli,
 		kubeCli:      kubeCli,
+		asCli:        asCli,
+		tcStsGetter:  helper.NewHijackClient(kubeCli, asCli).AppsV1(),
 		pdControl:    pdapi.NewDefaultPDControl(),
 		tidbControl:  controller.NewDefaultTiDBControl(),
 		pollInterval: pollInterval,
@@ -200,6 +207,8 @@ type OperatorActions interface {
 type operatorActions struct {
 	cli           versioned.Interface
 	kubeCli       kubernetes.Interface
+	asCli         asclientset.Interface
+	tcStsGetter   typedappsv1.StatefulSetsGetter
 	pdControl     pdapi.PDControlInterface
 	tidbControl   controller.TiDBControlInterface
 	pollInterval  time.Duration
@@ -222,21 +231,21 @@ type event struct {
 var _ = OperatorActions(&operatorActions{})
 
 type OperatorConfig struct {
-	Namespace                 string
-	ReleaseName               string
-	Image                     string
-	Tag                       string
-	SchedulerImage            string
-	SchedulerTag              string
-	SchedulerFeatures         []string
-	ControllerManagerFeatures []string
-	LogLevel                  string
-	WebhookServiceName        string
-	WebhookSecretName         string
-	WebhookConfigName         string
-	Context                   *apimachinery.CertContext
-	ImagePullPolicy           corev1.PullPolicy
-	TestMode                  bool
+	Namespace           string
+	ReleaseName         string
+	Image               string
+	Tag                 string
+	SchedulerImage      string
+	SchedulerTag        string
+	Features            []string
+	LogLevel            string
+	WebhookServiceName  string
+	WebhookSecretName   string
+	WebhookConfigName   string
+	Context             *apimachinery.CertContext
+	ImagePullPolicy     corev1.PullPolicy
+	TestMode            bool
+	AdvancedStatefulSet bool
 }
 
 type TidbClusterConfig struct {
@@ -360,11 +369,11 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string {
 	if oi.SchedulerTag != "" {
 		set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag
 	}
-	if len(oi.SchedulerFeatures) > 0 {
-		set["scheduler.features"] = fmt.Sprintf("{%s}", strings.Join(oi.SchedulerFeatures, ","))
+	if len(oi.Features) > 0 {
+		set["features"] = fmt.Sprintf("{%s}", strings.Join(oi.Features, ","))
 	}
-	if len(oi.ControllerManagerFeatures) > 0 {
-		set["controllerManager.features"] = fmt.Sprintf("{%s}", strings.Join(oi.ControllerManagerFeatures, ","))
+	if oi.AdvancedStatefulSet {
+		set["advancedStatefulset.create"] = "true"
 	}
 
 	arr := make([]string, 0, len(set))
@@ -392,6 +401,7 @@ func (oa *operatorActions) CleanCRDOrDie() {
 // InstallCRDOrDie install CRDs and wait for them to be established in Kubernetes.
 func (oa *operatorActions) InstallCRDOrDie() {
 	oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/crd.yaml"))
+	oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/advanced-statefulset-crd.v1beta1.yaml"))
 	out := oa.runKubectlOrDie([]string{"get", "crds", "--no-headers", `-ojsonpath={range .items[*]}{.metadata.name}{" "}{end}`}...)
 	waitArgs := []string{"wait", "--for=condition=Established"}
 	for _, crd := range strings.Split(out, " ") {
@@ -1051,7 +1061,7 @@ func (oa *operatorActions) CheckScaleInSafely(info *TidbClusterConfig) error {
 		}
 
 		tikvSetName := controller.TiKVMemberName(info.ClusterName)
-		tikvSet, err := oa.kubeCli.AppsV1().StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{})
+		tikvSet, err := oa.tcStsGetter.StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{})
 		if err != nil {
 			glog.Infof("failed to get tikvSet statefulset: [%s], error: %v", tikvSetName, err)
 			return false, nil
@@ -1266,7 +1276,7 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err
 	ns := tc.GetNamespace()
 	pdSetName := controller.PDMemberName(tcName)
 
-	pdSet, err := oa.kubeCli.AppsV1().StatefulSets(ns).Get(pdSetName, metav1.GetOptions{})
+	pdSet, err := oa.tcStsGetter.StatefulSets(ns).Get(pdSetName, metav1.GetOptions{})
 	if err != nil {
 		glog.Errorf("failed to get statefulset: %s/%s, %v", ns, pdSetName, err)
 		return false, nil
@@ -1331,7 +1341,7 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e
 	ns := tc.GetNamespace()
 	tikvSetName := controller.TiKVMemberName(tcName)
 
-	tikvSet, err := oa.kubeCli.AppsV1().StatefulSets(ns).Get(tikvSetName, metav1.GetOptions{})
+	tikvSet, err := oa.tcStsGetter.StatefulSets(ns).Get(tikvSetName, metav1.GetOptions{})
 	if err != nil {
 		glog.Errorf("failed to get statefulset: %s/%s, %v", ns, tikvSetName, err)
 		return false, nil
@@ -1390,7 +1400,7 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e
 	ns := tc.GetNamespace()
 	tidbSetName := controller.TiDBMemberName(tcName)
 
-	tidbSet, err := oa.kubeCli.AppsV1().StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{})
+	tidbSet, err := oa.tcStsGetter.StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{})
 	if err != nil {
 		glog.Errorf("failed to get statefulset: %s/%s, %v", ns, tidbSetName, err)
 		return false, nil
@@ -3134,7 +3144,7 @@ func (oa *operatorActions) checkManualPauseComponent(info *TidbClusterConfig, co
 
 	time.Sleep(30 * time.Second)
 
-	if set, err = oa.kubeCli.AppsV1().StatefulSets(ns).Get(setName, metav1.GetOptions{}); err != nil {
+	if set, err = oa.tcStsGetter.StatefulSets(ns).Get(setName, metav1.GetOptions{}); err != nil {
 		return fmt.Errorf("failed to get statefulset: [%s/%s], %v", ns, setName, err)
 	}