diff --git a/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml new file mode 100644 index 00000000000..b8c1898dbc8 --- /dev/null +++ b/charts/tidb-operator/templates/advanced-statefulset-deployment.yaml @@ -0,0 +1,55 @@ +{{- if .Values.advancedStatefulset.create }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: advanced-statefulset-controller + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +spec: + replicas: {{ .Values.advancedStatefulset.replicas }} + selector: + matchLabels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + template: + metadata: + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + spec: + serviceAccountName: {{ .Values.advancedStatefulset.serviceAccount }} + containers: + - name: advanced-statefulset-controller + image: {{ .Values.advancedStatefulset.image }} + imagePullPolicy: {{ .Values.advancedStatefulset.imagePullPolicy | default "IfNotPresent" }} + args: + - --v={{ .Values.advancedStatefulset.logLevel }} + - --leader-elect + - --leader-elect-resource-name=advanced-statefulset-controller + - --leader-elect-resource-namespace=$(POD_NAMESPACE) + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + resources: +{{ toYaml .Values.advancedStatefulset.resources | indent 12 }} + {{- with .Values.advancedStatefulset.nodeSelector }} + nodeSelector: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.advancedStatefulset.affinity }} + affinity: +{{ toYaml . | indent 8 }} + {{- end }} + {{- with .Values.advancedStatefulset.tolerations }} + tolerations: +{{ toYaml . | indent 8 }} + {{- end }} +{{- end }} diff --git a/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml new file mode 100644 index 00000000000..9458feae6ea --- /dev/null +++ b/charts/tidb-operator/templates/advanced-statefulset-rbac.yaml @@ -0,0 +1,96 @@ +{{- if .Values.advancedStatefulset.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.advancedStatefulset.serviceAccount }} + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: advanced-statefulset-controller + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +rules: +- apiGroups: + - apps.pingcap.com + resources: + - '*' + verbs: + - '*' +- apiGroups: + - 'apps' + resources: + - 'controllerrevisions' + verbs: + - '*' +- apiGroups: + - '' + resources: + - 'pods' + - 'persistentvolumeclaims' + - 'persistentvolumes' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: advanced-statefulset-controller + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: advanced-statefulset-controller +subjects: +- kind: ServiceAccount + name: advanced-statefulset-controller + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: advanced-statefulset-controller + namespace: {{ .Release.Namespace }} + labels: + app.kubernetes.io/name: {{ template "chart.name" . }} + app.kubernetes.io/managed-by: {{ .Release.Service }} + app.kubernetes.io/instance: {{ .Release.Name }} + app.kubernetes.io/component: advanced-statefulset-controller + helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version | replace "+" "_" }} +rules: +- apiGroups: + - '' + resources: + - 'endpoints' + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: advanced-statefulset-controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: advanced-statefulset-controller +subjects: +- kind: ServiceAccount + name: advanced-statefulset-controller + namespace: {{ .Release.Namespace }} +{{- end }} diff --git a/charts/tidb-operator/templates/controller-manager-deployment.yaml b/charts/tidb-operator/templates/controller-manager-deployment.yaml index d593c175165..a62844a6f04 100644 --- a/charts/tidb-operator/templates/controller-manager-deployment.yaml +++ b/charts/tidb-operator/templates/controller-manager-deployment.yaml @@ -49,6 +49,9 @@ spec: {{- if .Values.testMode }} - -test-mode={{ .Values.testMode }} {{- end}} + {{- if .Values.features }} + - -features={{ join "," .Values.features }} + {{- end }} env: - name: NAMESPACE valueFrom: diff --git a/charts/tidb-operator/templates/controller-manager-rbac.yaml b/charts/tidb-operator/templates/controller-manager-rbac.yaml index 1f19a596584..fac362938b1 100644 --- a/charts/tidb-operator/templates/controller-manager-rbac.yaml +++ b/charts/tidb-operator/templates/controller-manager-rbac.yaml @@ -56,6 +56,14 @@ rules: - restores - restores/finalizers verbs: ["*"] +{{- if .Values.features | has "AdvancedStatefulSet=true" }} +- apiGroups: + - apps.pingcap.com + resources: + - statefulsets + verbs: + - '*' +{{- end }} {{- end }} - apiGroups: [""] resources: ["nodes"] @@ -129,6 +137,14 @@ rules: - restores - restores/finalizers verbs: ["*"] +{{- if .Values.features | has "AdvancedStatefulSet=true" }} +- apiGroups: + - apps.pingcap.com + resources: + - statefulsets + verbs: + - '*' +{{- end }} --- kind: RoleBinding apiVersion: rbac.authorization.k8s.io/v1beta1 diff --git a/charts/tidb-operator/templates/scheduler-deployment.yaml b/charts/tidb-operator/templates/scheduler-deployment.yaml index fb3fe118c00..c4df791423f 100644 --- a/charts/tidb-operator/templates/scheduler-deployment.yaml +++ b/charts/tidb-operator/templates/scheduler-deployment.yaml @@ -35,8 +35,8 @@ spec: - /usr/local/bin/tidb-scheduler - -v={{ .Values.scheduler.logLevel }} - -port=10262 - {{- if .Values.scheduler.features }} - - -features={{ join "," .Values.scheduler.features }} + {{- if .Values.features }} + - -features={{ join "," .Values.features }} {{- end }} {{- if and (ne .Values.timezone "UTC") (ne .Values.timezone "") }} env: diff --git a/charts/tidb-operator/values.yaml b/charts/tidb-operator/values.yaml index 8a6afb09f6f..17e2dcd510a 100644 --- a/charts/tidb-operator/values.yaml +++ b/charts/tidb-operator/values.yaml @@ -21,6 +21,23 @@ defaultStorageClassName: local-storage # tidbBackupManagerImage: pingcap/tidb-backup-manager:latest # defaultBackupStorageClassName: local-storage +# +# Enable or disable tidb-operator features: +# +# StableScheduling (default: true) +# Enable stable scheduling of tidb servers. +# +# AdvancedStatefulSet (default: false) +# If enabled, tidb-operator will use AdvancedStatefulSet to manage pods +# instead of Kubernetes StatefulSet. +# It's ok to turn it on if this feature is not enabled. However it's not ok +# to turn it off when the tidb-operator already uses AdvancedStatefulSet to +# manage pods. This is in alpha phase. +# +features: [] +# - AdvancedStatefulSet=false +# - StableScheduling=true + controllerManager: # With rbac.create=false, the user is responsible for creating this account # With rbac.create=true, this service account will be created @@ -66,8 +83,6 @@ scheduler: logLevel: 2 replicas: 1 schedulerName: tidb-scheduler - # features: - # - StableScheduling=true resources: limits: cpu: 250m @@ -127,3 +142,36 @@ apiserver: # value: tidb-operator # effect: "NoSchedule" +# When AdvancedStatefulSet feature is enabled, you must install +# AdvancedStatefulSet controller. +# Note that AdvancedStatefulSet CRD must be installed manually via the following +# command: +# kubectl apply -f manifests/advanced-statefulset-crd.v1beta1.yaml +advancedStatefulset: + create: false + image: pingcap/advanced-statefulset:v0.1.0 + imagePullPolicy: IfNotPresent + serviceAccount: advanced-statefulset-controller + logLevel: 2 + replicas: 1 + resources: + limits: + cpu: 500m + memory: 300Mi + requests: + cpu: 200m + memory: 50Mi + ## affinity defines pod scheduling rules,affinity default settings is empty. + ## please read the affinity document before set your scheduling rule: + ## ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#affinity-and-anti-affinity + affinity: {} + ## nodeSelector ensure pods only assigning to nodes which have each of the indicated key-value pairs as labels + ## ref:https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#nodeselector + nodeSelector: {} + ## Tolerations are applied to pods, and allow pods to schedule onto nodes with matching taints. + ## refer to https://kubernetes.io/docs/concepts/configuration/taint-and-toleration + tolerations: [] + # - key: node-role + # operator: Equal + # value: tidb-operator + # effect: "NoSchedule" diff --git a/cmd/admission-controller/main.go b/cmd/admission-controller/main.go index 823d2542900..caf14e80c6d 100644 --- a/cmd/admission-controller/main.go +++ b/cmd/admission-controller/main.go @@ -20,9 +20,12 @@ import ( "os/signal" "syscall" + "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/version" "github.com/pingcap/tidb-operator/pkg/webhook" "k8s.io/apimachinery/pkg/util/wait" @@ -44,6 +47,7 @@ func init() { flag.BoolVar(&printVersion, "version", false, "Show version and quit") flag.StringVar(&certFile, "tlsCertFile", "/etc/webhook/certs/cert.pem", "File containing the x509 Certificate for HTTPS.") flag.StringVar(&keyFile, "tlsKeyFile", "/etc/webhook/certs/key.pem", "File containing the x509 private key to --tlsCertFile.") + features.DefaultFeatureGate.AddFlag(flag.CommandLine) flag.Parse() } @@ -68,10 +72,22 @@ func main() { glog.Fatalf("failed to create Clientset: %v", err) } - kubeCli, err := kubernetes.NewForConfig(cfg) + var kubeCli kubernetes.Interface + kubeCli, err = kubernetes.NewForConfig(cfg) if err != nil { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + asCli, err := asclientset.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err) + } + + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + // If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use + // AdvancedStatefulSet. + kubeCli = helper.NewHijackClient(kubeCli, asCli) + } + informerFactory := informers.NewSharedInformerFactory(cli, controller.ResyncDuration) kubeInformerFactory := kubeinformers.NewSharedInformerFactory(kubeCli, controller.ResyncDuration) diff --git a/cmd/controller-manager/main.go b/cmd/controller-manager/main.go index ac745548325..53cf5454b39 100644 --- a/cmd/controller-manager/main.go +++ b/cmd/controller-manager/main.go @@ -21,6 +21,8 @@ import ( "os" "time" + "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" informers "github.com/pingcap/tidb-operator/pkg/client/informers/externalversions" "github.com/pingcap/tidb-operator/pkg/controller" @@ -28,6 +30,7 @@ import ( "github.com/pingcap/tidb-operator/pkg/controller/backupschedule" "github.com/pingcap/tidb-operator/pkg/controller/restore" "github.com/pingcap/tidb-operator/pkg/controller/tidbcluster" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" @@ -68,6 +71,7 @@ func init() { flag.DurationVar(&controller.ResyncDuration, "resync-duration", time.Duration(30*time.Second), "Resync time of informer") flag.BoolVar(&controller.TestMode, "test-mode", false, "whether tidb-operator run in test mode") flag.StringVar(&controller.TidbBackupManagerImage, "tidb-backup-manager-image", "pingcap/tidb-backup-manager:latest", "The image of backup manager tool") + features.DefaultFeatureGate.AddFlag(flag.CommandLine) flag.Parse() } @@ -101,10 +105,21 @@ func main() { if err != nil { glog.Fatalf("failed to create Clientset: %v", err) } - kubeCli, err := kubernetes.NewForConfig(cfg) + var kubeCli kubernetes.Interface + kubeCli, err = kubernetes.NewForConfig(cfg) if err != nil { glog.Fatalf("failed to get kubernetes Clientset: %v", err) } + asCli, err := asclientset.NewForConfig(cfg) + if err != nil { + glog.Fatalf("failed to get advanced-statefulset Clientset: %v", err) + } + + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + // If AdvancedStatefulSet is enabled, we hijack the Kubernetes client to use + // AdvancedStatefulSet. + kubeCli = helper.NewHijackClient(kubeCli, asCli) + } var informerFactory informers.SharedInformerFactory var kubeInformerFactory kubeinformers.SharedInformerFactory diff --git a/go.mod b/go.mod index e458a0e7836..2810330f1d8 100644 --- a/go.mod +++ b/go.mod @@ -45,6 +45,7 @@ require ( github.com/onsi/gomega v1.5.0 github.com/opentracing/opentracing-go v1.1.0 // indirect github.com/pierrec/lz4 v2.0.5+incompatible // indirect + github.com/pingcap/advanced-statefulset v0.1.0 github.com/pingcap/errors v0.11.0 github.com/pingcap/kvproto v0.0.0-20190516013202-4cf58ad90b6c github.com/pingcap/log v0.0.0-20190715063458-479153f07ebd diff --git a/go.sum b/go.sum index 6082e297f73..f4c71230597 100644 --- a/go.sum +++ b/go.sum @@ -569,6 +569,8 @@ github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+v github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/advanced-statefulset v0.1.0 h1:rjUx6Tc90YwlRrgIXWTvjr8D97fx0GP4E49dObmBqj0= +github.com/pingcap/advanced-statefulset v0.1.0/go.mod h1:rg2p1v6AGsKhvEZi6Sm0YNYJCmdXdZZhQ6Sviei7Ivs= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8 h1:USx2/E1bX46VG32FIw034Au6seQ2fY9NEILmNh/UlQg= github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= @@ -816,6 +818,8 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c h1:fqgJT0MGcGpPgpWU7VRdRjuArfcOvC4AoJmILihzhDg= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= +golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20170824195420-5d2fd3ccab98/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181003024731-2f84ea8ef872/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/manifests/advanced-statefulset-crd.v1beta1.yaml b/manifests/advanced-statefulset-crd.v1beta1.yaml new file mode 100644 index 00000000000..8b0c4fca8f0 --- /dev/null +++ b/manifests/advanced-statefulset-crd.v1beta1.yaml @@ -0,0 +1,51 @@ +# For Kubernetes before 1.16. +# TODO more validations, etc. +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: statefulsets.apps.pingcap.com +spec: + group: apps.pingcap.com + version: v1alpha1 + scope: Namespaced + names: + plural: statefulsets + singular: statefulset + kind: StatefulSet + shortNames: + - asts + validation: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + replicas: + type: integer + minimum: 0 + selector: + type: object + serviceName: + type: string + template: + type: object + revisionHistoryLimit: + type: integer + minimum: 1 + versions: + - name: v1alpha1 + served: true + storage: true + # subresources describes the subresources for custom resources. + subresources: + # status enables the status subresource. + status: {} + # scale enables the scale subresource. + scale: + # specReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Spec.Replicas. + specReplicasPath: .spec.replicas + # statusReplicasPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Replicas. + statusReplicasPath: .status.replicas + # labelSelectorPath defines the JSONPath inside of a custom resource that corresponds to Scale.Status.Selector. + labelSelectorPath: .status.labelSelector diff --git a/manifests/webhook.yaml b/manifests/webhook.yaml index 62c67c86164..aaea2ac6110 100644 --- a/manifests/webhook.yaml +++ b/manifests/webhook.yaml @@ -21,6 +21,12 @@ rules: - apiGroups: ["pingcap.com"] resources: ["tidbclusters"] verbs: ["*"] + - apiGroups: + - apps.pingcap.com + resources: + - statefulsets + verbs: + - '*' --- apiVersion: v1 kind: ServiceAccount @@ -86,7 +92,10 @@ spec: - /usr/local/bin/tidb-admission-controller - -tlsCertFile=/etc/webhook/certs/cert.pem - -tlsKeyFile=/etc/webhook/certs/key.pem - - -v=2 + - -v=4 + # TODO webhook is not bundled with tidb-operator yet and tested in + # e2e only, make it configurable in helm chart + - -features=AdvancedStatefulSet=true volumeMounts: - name: webhook-certs mountPath: /etc/webhook/certs @@ -116,3 +125,7 @@ webhooks: apiGroups: [ "apps", "" ] apiVersions: ["v1beta1", "v1"] resources: ["statefulsets"] + - operations: [ "UPDATE" ] + apiGroups: [ "apps.pingcap.com"] + apiVersions: ["v1alpha1"] + resources: ["statefulsets"] diff --git a/pkg/features/features.go b/pkg/features/features.go index 4c40bccf507..32f2276627b 100644 --- a/pkg/features/features.go +++ b/pkg/features/features.go @@ -25,7 +25,8 @@ import ( var ( allFeatures = sets.NewString(StableScheduling) defaultFeatures = map[string]bool{ - StableScheduling: true, + StableScheduling: true, + AdvancedStatefulSet: false, } // DefaultFeatureGate is a shared global FeatureGate. DefaultFeatureGate FeatureGate = NewFeatureGate() @@ -34,6 +35,9 @@ var ( const ( // StableScheduling controls stable scheduling of TiDB members. StableScheduling string = "StableScheduling" + + // AdvancedStatefulSet controls whether to use AdvancedStatefulSet to manage pods + AdvancedStatefulSet string = "AdvancedStatefulSet" ) type FeatureGate interface { diff --git a/pkg/webhook/statefulset/statefulset.go b/pkg/webhook/statefulset/statefulset.go index 09133d6bea1..c4508dd8471 100644 --- a/pkg/webhook/statefulset/statefulset.go +++ b/pkg/webhook/statefulset/statefulset.go @@ -18,8 +18,10 @@ import ( "fmt" "strconv" + asappsv1alpha1 "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/pkg/features" "github.com/pingcap/tidb-operator/pkg/label" "github.com/pingcap/tidb-operator/pkg/webhook/util" "k8s.io/api/admission/v1beta1" @@ -44,15 +46,19 @@ func AdmitStatefulSets(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { name := ar.Request.Name namespace := ar.Request.Namespace - glog.V(4).Infof("admit statefulsets [%s/%s]", namespace, name) + expectedGroup := "apps" + if features.DefaultFeatureGate.Enabled(features.AdvancedStatefulSet) { + expectedGroup = asappsv1alpha1.GroupName + } apiVersion := ar.Request.Resource.Version - setResource := metav1.GroupVersionResource{Group: "apps", Version: apiVersion, Resource: "statefulsets"} - if ar.Request.Resource.Group != "apps" || ar.Request.Resource.Resource != "statefulsets" { + setResource := metav1.GroupVersionResource{Group: expectedGroup, Version: apiVersion, Resource: "statefulsets"} + if ar.Request.Resource.Group != setResource.Group || ar.Request.Resource.Resource != setResource.Resource { err := fmt.Errorf("expect resource to be %s instead of %s", setResource, ar.Request.Resource) glog.Error(err) return util.ARFail(err) } + glog.V(4).Infof("admit %s [%s/%s]", setResource, namespace, name) if versionCli == nil { cfg, err := rest.InClusterConfig() diff --git a/tests/actions.go b/tests/actions.go index f0211ca2578..810c7fdb833 100644 --- a/tests/actions.go +++ b/tests/actions.go @@ -32,9 +32,12 @@ import ( "sync" "time" - "github.com/ghodss/yaml" // To register MySQL driver _ "github.com/go-sql-driver/mysql" + + "github.com/ghodss/yaml" + "github.com/pingcap/advanced-statefulset/pkg/apis/apps/v1alpha1/helper" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" pingcapErrors "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/tidb-operator/pkg/apis/pingcap/v1alpha1" @@ -58,6 +61,7 @@ import ( "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + typedappsv1 "k8s.io/client-go/kubernetes/typed/apps/v1" glog "k8s.io/klog" ) @@ -76,12 +80,15 @@ const ( func NewOperatorActions(cli versioned.Interface, kubeCli kubernetes.Interface, + asCli asclientset.Interface, pollInterval time.Duration, cfg *Config, clusters []*TidbClusterConfig) OperatorActions { oa := &operatorActions{ cli: cli, kubeCli: kubeCli, + asCli: asCli, + tcStsGetter: helper.NewHijackClient(kubeCli, asCli).AppsV1(), pdControl: pdapi.NewDefaultPDControl(), tidbControl: controller.NewDefaultTiDBControl(), pollInterval: pollInterval, @@ -200,6 +207,8 @@ type OperatorActions interface { type operatorActions struct { cli versioned.Interface kubeCli kubernetes.Interface + asCli asclientset.Interface + tcStsGetter typedappsv1.StatefulSetsGetter pdControl pdapi.PDControlInterface tidbControl controller.TiDBControlInterface pollInterval time.Duration @@ -223,24 +232,25 @@ type event struct { var _ = OperatorActions(&operatorActions{}) type OperatorConfig struct { - Namespace string - ReleaseName string - Image string - Tag string - SchedulerImage string - SchedulerTag string - SchedulerFeatures []string - LogLevel string - WebhookServiceName string - WebhookSecretName string - WebhookConfigName string - Context *apimachinery.CertContext - ImagePullPolicy corev1.PullPolicy - TestMode bool - ApiServerImage string - ApiServerCert string - ApiServerKey string - ApiServerCaBundle string + Namespace string + ReleaseName string + Image string + Tag string + SchedulerImage string + SchedulerTag string + Features []string + LogLevel string + WebhookServiceName string + WebhookSecretName string + WebhookConfigName string + Context *apimachinery.CertContext + ImagePullPolicy corev1.PullPolicy + TestMode bool + ApiServerImage string + ApiServerCert string + ApiServerKey string + ApiServerCaBundle string + AdvancedStatefulSet bool } type TidbClusterConfig struct { @@ -364,8 +374,11 @@ func (oi *OperatorConfig) OperatorHelmSetString(m map[string]string) string { if oi.SchedulerTag != "" { set["scheduler.kubeSchedulerImageTag"] = oi.SchedulerTag } - if len(oi.SchedulerFeatures) > 0 { - set["scheduler.features"] = fmt.Sprintf("{%s}", strings.Join(oi.SchedulerFeatures, ",")) + if len(oi.Features) > 0 { + set["features"] = fmt.Sprintf("{%s}", strings.Join(oi.Features, ",")) + } + if oi.AdvancedStatefulSet { + set["advancedStatefulset.create"] = "true" } arr := make([]string, 0, len(set)) @@ -394,6 +407,7 @@ func (oa *operatorActions) CleanCRDOrDie() { func (oa *operatorActions) InstallCRDOrDie() { oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/crd.yaml")) oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/data-resource-crd.yaml")) + oa.runKubectlOrDie("apply", "-f", oa.manifestPath("e2e/advanced-statefulset-crd.v1beta1.yaml")) out := oa.runKubectlOrDie([]string{"get", "crds", "--no-headers", `-ojsonpath={range .items[*]}{.metadata.name}{" "}{end}`}...) waitArgs := []string{"wait", "--for=condition=Established"} for _, crd := range strings.Split(out, " ") { @@ -1053,7 +1067,7 @@ func (oa *operatorActions) CheckScaleInSafely(info *TidbClusterConfig) error { } tikvSetName := controller.TiKVMemberName(info.ClusterName) - tikvSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{}) + tikvSet, err := oa.tcStsGetter.StatefulSets(info.Namespace).Get(tikvSetName, metav1.GetOptions{}) if err != nil { glog.Infof("failed to get tikvSet statefulset: [%s], error: %v", tikvSetName, err) return false, nil @@ -1268,7 +1282,7 @@ func (oa *operatorActions) pdMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, err ns := tc.GetNamespace() pdSetName := controller.PDMemberName(tcName) - pdSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(ns).Get(pdSetName, metav1.GetOptions{}) + pdSet, err := oa.tcStsGetter.StatefulSets(ns).Get(pdSetName, metav1.GetOptions{}) if err != nil { glog.Errorf("failed to get statefulset: %s/%s, %v", ns, pdSetName, err) return false, nil @@ -1333,7 +1347,7 @@ func (oa *operatorActions) tikvMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns := tc.GetNamespace() tikvSetName := controller.TiKVMemberName(tcName) - tikvSet, err := oa.kubeCli.AppsV1().StatefulSets(ns).Get(tikvSetName, metav1.GetOptions{}) + tikvSet, err := oa.tcStsGetter.StatefulSets(ns).Get(tikvSetName, metav1.GetOptions{}) if err != nil { glog.Errorf("failed to get statefulset: %s/%s, %v", ns, tikvSetName, err) return false, nil @@ -1392,7 +1406,7 @@ func (oa *operatorActions) tidbMembersReadyFn(tc *v1alpha1.TidbCluster) (bool, e ns := tc.GetNamespace() tidbSetName := controller.TiDBMemberName(tcName) - tidbSet, err := oa.kubeCli.AppsV1beta1().StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{}) + tidbSet, err := oa.tcStsGetter.StatefulSets(ns).Get(tidbSetName, metav1.GetOptions{}) if err != nil { glog.Errorf("failed to get statefulset: %s/%s, %v", ns, tidbSetName, err) return false, nil @@ -3150,7 +3164,7 @@ func (oa *operatorActions) checkManualPauseComponent(info *TidbClusterConfig, co time.Sleep(30 * time.Second) - if set, err = oa.kubeCli.AppsV1().StatefulSets(ns).Get(setName, metav1.GetOptions{}); err != nil { + if set, err = oa.tcStsGetter.StatefulSets(ns).Get(setName, metav1.GetOptions{}); err != nil { return fmt.Errorf("failed to get statefulset: [%s/%s], %v", ns, setName, err) } diff --git a/tests/cmd/e2e/main.go b/tests/cmd/e2e/main.go index 465c60e93cc..0e1cf0bbbcc 100644 --- a/tests/cmd/e2e/main.go +++ b/tests/cmd/e2e/main.go @@ -54,8 +54,12 @@ func main() { } go tests.StartValidatingAdmissionWebhookServerOrDie(certCtx) - restConfig := client.GetConfigOrDie() - cli, kubeCli := client.NewCliOrDie() + restConfig, err := client.GetConfig() + if err != nil { + panic(err) + } + cli, kubeCli, asCli := client.NewCliOrDie() + ocfg := newOperatorConfig() cluster1 := newTidbClusterConfig(ns, "cluster1", "", "") @@ -66,7 +70,7 @@ func main() { cluster5 := newTidbClusterConfig(ns, "cluster5", "", "v2.1.16") // for v2.1.x series cluster5.Resources["tikv.resources.limits.storage"] = "1G" - oa := tests.NewOperatorActions(cli, kubeCli, tests.DefaultPollInterval, cfg, nil) + oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, cfg, nil) oa.CleanCRDOrDie() oa.InstallCRDOrDie() oa.LabelNodesOrDie() @@ -253,15 +257,17 @@ func newOperatorConfig() *tests.OperatorConfig { Tag: cfg.OperatorTag, SchedulerImage: "mirantis/hypokube", SchedulerTag: "final", - SchedulerFeatures: []string{ + Features: []string{ "StableScheduling=true", + "AdvancedStatefulSet=true", }, - LogLevel: "2", - WebhookServiceName: "webhook-service", - WebhookSecretName: "webhook-secret", - WebhookConfigName: "webhook-config", - ImagePullPolicy: v1.PullIfNotPresent, - TestMode: true, + LogLevel: "4", + WebhookServiceName: "webhook-service", + WebhookSecretName: "webhook-secret", + WebhookConfigName: "webhook-config", + ImagePullPolicy: v1.PullIfNotPresent, + TestMode: true, + AdvancedStatefulSet: true, } } diff --git a/tests/cmd/stability/main.go b/tests/cmd/stability/main.go index fad934c13cf..fb613e319e3 100644 --- a/tests/cmd/stability/main.go +++ b/tests/cmd/stability/main.go @@ -67,7 +67,7 @@ func main() { } func run() { - cli, kubeCli := client.NewCliOrDie() + cli, kubeCli, asCli := client.NewCliOrDie() ocfg := newOperatorConfig() @@ -109,7 +109,7 @@ func run() { fta := tests.NewFaultTriggerAction(cli, kubeCli, cfg) fta.CheckAndRecoverEnvOrDie() - oa := tests.NewOperatorActions(cli, kubeCli, tests.DefaultPollInterval, cfg, allClusters) + oa := tests.NewOperatorActions(cli, kubeCli, asCli, tests.DefaultPollInterval, cfg, allClusters) oa.CheckK8sAvailableOrDie(nil, nil) oa.LabelNodesOrDie() @@ -381,7 +381,7 @@ func newOperatorConfig() *tests.OperatorConfig { Image: cfg.OperatorImage, Tag: cfg.OperatorTag, SchedulerImage: "gcr.io/google-containers/hyperkube", - SchedulerFeatures: []string{ + Features: []string{ "StableScheduling=true", }, LogLevel: "2", diff --git a/tests/failover.go b/tests/failover.go index 99006e7acc3..148cb78138a 100644 --- a/tests/failover.go +++ b/tests/failover.go @@ -152,7 +152,7 @@ func (oa *operatorActions) CheckFailoverPending(info *TidbClusterConfig, node st glog.Infof("pending failover,failed to get tidbcluster:[%s], error: %v", info.FullName(), err) if strings.Contains(err.Error(), "Client.Timeout exceeded while awaiting headers") { glog.Info("create new client") - newCli, _ := client.NewCliOrDie() + newCli, _, _ := client.NewCliOrDie() oa.cli = newCli } return false, nil diff --git a/tests/manifests/e2e/e2e.yaml b/tests/manifests/e2e/e2e.yaml index 37892faeada..6200fb48cfd 100644 --- a/tests/manifests/e2e/e2e.yaml +++ b/tests/manifests/e2e/e2e.yaml @@ -47,8 +47,8 @@ spec: command: - /usr/local/bin/e2e - --operator-tag=e2e - - --operator-image=pingcap/tidb-operator:latest - - --test-apiserver-image=pingcap/test-apiserver:latest + - --operator-image=localhost:5000/pingcap/tidb-operator:latest + - --test-apiserver-image=localhost:5000/pingcap/test-apiserver:latest - --tidb-versions=v3.0.2,v3.0.3,v3.0.4 - --chart-dir=/charts volumeMounts: diff --git a/tests/pkg/client/client.go b/tests/pkg/client/client.go index 982db2803c1..003229a0879 100644 --- a/tests/pkg/client/client.go +++ b/tests/pkg/client/client.go @@ -10,6 +10,7 @@ import ( "github.com/pingcap/tidb-operator/tests/slack" "github.com/juju/errors" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned/typed/pingcap/v1alpha1" "k8s.io/client-go/kubernetes" @@ -30,7 +31,7 @@ func init() { "Only required if out-of-cluster.") } -func NewCliOrDie() (versioned.Interface, kubernetes.Interface) { +func NewCliOrDie() (versioned.Interface, kubernetes.Interface, asclientset.Interface) { cfg, err := GetConfig() if err != nil { slack.NotifyAndPanic(err) @@ -117,7 +118,7 @@ func LoadConfig() (*rest.Config, error) { return cfg, errors.Trace(err) } -func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interface) { +func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interface, asclientset.Interface) { cfg.Timeout = 30 * time.Second cli, err := versioned.NewForConfig(cfg) if err != nil { @@ -129,5 +130,10 @@ func buildClientsOrDie(cfg *rest.Config) (versioned.Interface, kubernetes.Interf slack.NotifyAndPanic(err) } - return cli, kubeCli + asCli, err := asclientset.NewForConfig(cfg) + if err != nil { + slack.NotifyAndPanic(err) + } + + return cli, kubeCli, asCli } diff --git a/tests/pkg/webhook/pods.go b/tests/pkg/webhook/pods.go index 42c88f9ef1a..c1de4298e9a 100644 --- a/tests/pkg/webhook/pods.go +++ b/tests/pkg/webhook/pods.go @@ -31,7 +31,7 @@ func admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionResponse { return toAdmissionResponse(err) } - versionCli, kubeCli := client.NewCliOrDie() + versionCli, kubeCli, _ := client.NewCliOrDie() name := ar.Request.Name namespace := ar.Request.Namespace