Add periodical e2e job for GKE

ci/e2e_gke.groovy

@@ -0,0 +1,136 @@
+//
+// Jenkins pipeline for GKE e2e job.
+//
+// This script is written in declarative syntax. Refer to
+// https://jenkins.io/doc/book/pipeline/syntax/ for more details.
+//
+// Note that parameters of the job is configured in this script.
+//
+
+import groovy.transform.Field
+
+@Field
+def podYAML = '''
+apiVersion: v1
+kind: Pod
+spec:
+  containers:
+  - name: main
+    image: gcr.io/k8s-testimages/kubekins-e2e:v20191108-9467d02-master
+    command:
+    - runner.sh
+    - sleep
+    - 1d
+    # we need privileged mode in order to do docker in docker
+    securityContext:
+      privileged: true
+    env:
+    - name: DOCKER_IN_DOCKER_ENABLED
+      value: "true"
+    resources:
+      requests:
+        memory: "4000Mi"
+        cpu: 2000m
+    volumeMounts:
+    # dind expects /var/lib/docker to be volume
+    - name: docker-root
+      mountPath: /var/lib/docker
+  volumes:
+  - name: docker-root
+    emptyDir: {}
+'''
+
+pipeline {
+    agent {
+        kubernetes {
+            yaml podYAML
+            defaultContainer "main"
+            customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator"
+        }
+    }
+
+    parameters {
+        string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url')
+        string(name: 'GIT_REF', defaultValue: 'master', description: 'git ref spec to checkout, e.g. master, release-1.1')
+        string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889')
+        string(name: 'CLUSTER', defaultValue: 'jenkins-tidb-operator-e2e', description: 'the name of the cluster')
+        string(name: 'GCP_PROJECT', defaultValue: 'smooth-tendril-207212', description: 'the GCP project ID')
+        string(name: 'GCP_ZONE', defaultValue: 'us-central1-b', description: 'the GCP zone')
+        string(name: 'GINKGO_NODES', defaultValue: '8', description: 'the number of ginkgo nodes')
+    }
+
+    environment {
+        GIT_REF = ''
+        ARTIFACTS = "${env.WORKSPACE}/artifacts"
+    }
+
+    stages {
+        stage("Prepare") {
+            steps {
+                // The declarative model for Jenkins Pipelines has a restricted
+                // subset of syntax that it allows in the stage blocks. We use
+                // script step to bypass the restriction.
+                // https://jenkins.io/doc/book/pipeline/syntax/#script
+                script {
+                    GIT_REF = params.GIT_REF
+                    if (params.PR_ID != "") {
+                        GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head"
+                    }
+                }
+                echo "env.NODE_NAME: ${env.NODE_NAME}"
+                echo "env.WORKSPACE: ${env.WORKSPACE}"
+                echo "GIT_REF: ${GIT_REF}"
+                echo "ARTIFACTS: ${ARTIFACTS}"
+            }
+        }
+
+        stage("Checkout") {
+            steps {
+                checkout scm: [
+                        $class: 'GitSCM',
+                        branches: [[name: GIT_REF]],
+                        userRemoteConfigs: [[
+                            credentialsId: 'github-sre-bot-ssh',
+                            refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*',
+                            url: "${params.GIT_URL}",
+                        ]]
+                    ]
+            }
+        }
+
+        stage("Run") {
+            steps {
+                withCredentials([
+                    file(credentialsId: 'TIDB_OPERATOR_GCP_CREDENTIALS', variable: 'GCP_CREDENTIALS'),
+                    file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PRIVATE_KEY', variable: 'GCP_SSH_PRIVATE_KEY'),
+                    file(credentialsId: 'TIDB_OPERATOR_GCP_SSH_PUBLIC_KEY', variable: 'GCP_SSH_PUBLIC_KEY'),
+                ]) {
+                    sh """
+                    #!/bin/bash
+                    export PROVIDER=gke
+                    export CLUSTER=${params.CLUSTER}
+                    export GCP_ZONE=${params.GCP_ZONE}
+                    export GCP_PROJECT=${params.GCP_PROJECT}
+                    export GINKGO_NODES=${params.GINKGO_NODES}
+                    export REPORT_DIR=${ARTIFACTS}
+                    echo "info: try to clean the cluster created previously"
+                    SKIP_BUILD=y SKIP_IMAGE_BUILD=y SKIP_UP=y SKIP_TEST=y ./hack/e2e.sh
+                    echo "info: begin to run e2e"
+                    ./hack/e2e.sh -- --ginkgo.skip='\\[Serial\\]' --ginkgo.focus='\\[tidb-operator\\]'
+                    """
+                }
+            }
+        }
+    }
+
+    post {
+        always {
+            dir(ARTIFACTS) {
+                archiveArtifacts artifacts: "**", allowEmptyArchive: true
+                junit testResults: "*.xml", allowEmptyResults: true
+            }
+        }
+    }
+}
+
+// vim: et sw=4 ts=4

hack/e2e.sh

@@ -64,11 +64,12 @@ Environments:
     QUAY_IO_MIRROR        configure mirror for quay.io
     KIND_DATA_HOSTPATH    (kind only) the host path of data directory for kind cluster, defaults: none
     GCP_PROJECT           (gke only) the GCP project to run in
-    GCP_SERVICE_ACCOUNT   (gke only) the GCP service account to use
+    GCP_CREDENTIALS       (gke only) the GCP service account to use
     GCP_REGION            (gke only) the GCP region, if specified a regional cluster is creaetd
     GCP_ZONE              (gke only) the GCP zone, if specified a zonal cluster is created
     GCP_SSH_PRIVATE_KEY   (gke only) the path to the private ssh key
     GCP_SSH_PUBLIC_KEY    (gke only) the path to the public ssh key
+    GCP_MACHINE_TYPE      (gke only) the machine type of instance, defaults: n1-standard-4
     AWS_ACCESS_KEY_ID     (eks only) the aws access key id
     AWS_SECRET_ACCESS_KEY (eks only) the aws secret access key
     AWS_REGION            (eks only) the aws region
@@ -115,6 +116,7 @@ Examples:
         - Kubernetes Engine Admin
         - Service Account User
         - Storage Admin
+        - Compute Instance Admin (v1)
 
     You can create ssh keypair with ssh-keygen at  ~/.ssh/google_compute_engine
     or specifc existing ssh keypair with following environments:
@@ -125,7 +127,7 @@ Examples:
     Then run with following additional GCP-specific environments:
 
         export GCP_PROJECT=<project>
-        export GCP_SERVICE_ACCOUNT=<path-to-gcp-service-account>
+        export GCP_CREDENTIALS=<path-to-gcp-service-account>
         export GCP_ZONE=us-central1-b
 
         PROVIDER=gke ./hack/e2e.sh -- <e2e args>
@@ -186,11 +188,12 @@ SKIP_TEST=${SKIP_TEST:-}
 REUSE_CLUSTER=${REUSE_CLUSTER:-}
 KIND_DATA_HOSTPATH=${KIND_DATA_HOSTPATH:-none}
 GCP_PROJECT=${GCP_PROJECT:-}
-GCP_SERVICE_ACCOUNT=${GCP_SERVICE_ACCOUNT:-}
+GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
 GCP_REGION=${GCP_REGION:-}
 GCP_ZONE=${GCP_ZONE:-}
 GCP_SSH_PRIVATE_KEY=${GCP_SSH_PRIVATE_KEY:-}
 GCP_SSH_PUBLIC_KEY=${GCP_SSH_PUBLIC_KEY:-}
+GCP_MACHINE_TYPE=${GCP_MACHINE_TYPE:-n1-standard-4}
 AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID:-}
 AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY:-}
 AWS_REGION=${AWS_REGION:-}
@@ -213,7 +216,7 @@ echo "SKIP_UP: $SKIP_UP"
 echo "SKIP_DOWN: $SKIP_DOWN"
 echo "KIND_DATA_HOSTPATH: $KIND_DATA_HOSTPATH"
 echo "GCP_PROJECT: $GCP_PROJECT"
-echo "GCP_SERVICE_ACCOUNT: $GCP_SERVICE_ACCOUNT"
+echo "GCP_CREDENTIALS: $GCP_CREDENTIALS"
 echo "GCP_REGION: $GCP_REGION"
 echo "GCP_ZONE: $GCP_ZONE"
 echo "AWS_ACCESS_KEY_ID: $AWS_ACCESS_KEY_ID"
@@ -415,8 +418,8 @@ elif [ "$PROVIDER" == "gke" ]; then
         echo "error: GCP_PROJECT is required"
         exit 1
     fi
-    if [ -z "$GCP_SERVICE_ACCOUNT" ]; then
-        echo "error: GCP_SERVICE_ACCOUNT is required"
+    if [ -z "$GCP_CREDENTIALS" ]; then
+        echo "error: GCP_CREDENTIALS is required"
         exit 1
     fi
     if [ -z "$GCP_REGION" -a -z "$GCP_ZONE" ]; then
@@ -430,21 +433,25 @@ elif [ "$PROVIDER" == "gke" ]; then
     if [ ! -d ~/.ssh ]; then
         mkdir ~/.ssh
     fi
-    if [ ! -e ~/.ssh/google_compute_engine -o -n "$GCP_SSH_PRIVATE_KEY" ]; then
+    if [ ! -e ~/.ssh/google_compute_engine -a -n "$GCP_SSH_PRIVATE_KEY" ]; then
         echo "Copying $GCP_SSH_PRIVATE_KEY to ~/.ssh/google_compute_engine" >&2
         cp $GCP_SSH_PRIVATE_KEY ~/.ssh/google_compute_engine
         chmod 0600 ~/.ssh/google_compute_engine
     fi
-    if [ ! -e ~/.ssh/google_compute_engine.pub -o -n "$GCP_SSH_PUBLIC_KEY" ]; then
+    if [ ! -e ~/.ssh/google_compute_engine.pub -a -n "$GCP_SSH_PUBLIC_KEY" ]; then
         echo "Copying $GCP_SSH_PUBLIC_KEY to ~/.ssh/google_compute_engine.pub" >&2
         cp $GCP_SSH_PUBLIC_KEY ~/.ssh/google_compute_engine.pub
         chmod 0600 ~/.ssh/google_compute_engine.pub
     fi
+    ! read -r -d '' nodePoolsJSON <<EOF
+{"default":{"Nodes":${KUBE_WORKERS},"MachineType":"${GCP_MACHINE_TYPE}"}}
+EOF
     kubetest2_args+=(
         --cluster-name "$CLUSTER"
         --project "$GCP_PROJECT"
-        --gcp-service-account "$GCP_SERVICE_ACCOUNT"
+        --gcp-service-account "$GCP_CREDENTIALS"
         --environment prod
+        --node-pools "$nodePoolsJSON"
     )
     if [ -n "$GCP_REGION" ]; then
         kubetest2_args+=(
@@ -498,12 +505,15 @@ export PROVIDER
 export CLUSTER
 export KUBECONFIG
 export GCP_PROJECT
+export GCP_REGION
+export GCP_ZONE
+export GCP_CREDENTIALS
 export IMAGE_TAG
 export SKIP_GINKGO
 export SKIP_IMAGE_LOAD
 export TIDB_OPERATOR_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator:${IMAGE_TAG}
 export E2E_IMAGE=$DOCKER_REGISTRY/pingcap/tidb-operator-e2e:${IMAGE_TAG}
-export PATH=$PATH:$OUTPUT_BIN
+export PATH=$OUTPUT_BIN:$PATH
 
 # Environments for kubetest2
 if [ -n "${REPORT_DIR:-}" ]; then

hack/lib.sh

@@ -34,7 +34,7 @@ HELM_BIN=$OUTPUT_BIN/helm
 HELM_VERSION=${HELM_VERSION:-2.9.1}
 KIND_VERSION=${KIND_VERSION:-0.7.0}
 KIND_BIN=$OUTPUT_BIN/kind
-KUBETEST2_VERSION=v0.0.7
+KUBETEST2_VERSION=v0.0.8
 KUBETSTS2_BIN=$OUTPUT_BIN/kubetest2
 AWS_K8S_TESTER_VERSION=v0.6.2
 AWS_K8S_TESTER_BIN=$OUTPUT_BIN/aws-k8s-tester

hack/run-e2e.sh

@@ -25,6 +25,9 @@ source $ROOT/hack/lib.sh
 PROVIDER=${PROVIDER:-}
 CLUSTER=${CLUSTER:-}
 GCP_PROJECT=${GCP_PROJECT:-}
+GCP_REGION=${GCP_REGION:-}
+GCP_ZONE=${GCP_ZONE:-}
+GCP_CREDENTIALS=${GCP_CREDENTIALS:-}
 IMAGE_TAG=${IMAGE_TAG:-}
 SKIP_IMAGE_LOAD=${SKIP_IMAGE_LOAD:-}
 TIDB_OPERATOR_IMAGE=${TIDB_OPERATOR_IMAGE:-localhost:5000/pingcap/tidb-operator:latest}
@@ -119,8 +122,8 @@ EOF
     elif [ "$PROVIDER" == "gke" ]; then
         # disks are created under /mnt/stateful_partition directory
         # https://cloud.google.com/container-optimized-os/docs/concepts/disks-and-filesystem
-        for n in $($KUBECTL_BIN --context $KUBECONTEXT get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
-            gcloud compute ssh $n --command 'sudo bash -c '"'"'
+        for n in $($KUBECTL_BIN --context "$KUBECONTEXT" get nodes -ojsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'); do
+            gcloud compute ssh e2e@$n --command 'sudo bash -c '"'"'
 test -d /mnt/stateful_partition/disks || mkdir -p /mnt/stateful_partition/disks
 df -h /mnt/stateful_partition/disks
 test -d /mnt/disks || mkdir -p /mnt/disks
@@ -258,9 +261,28 @@ function e2e::image_load() {
 hack::ensure_kubectl
 hack::ensure_helm
 
+if [ "$PROVIDER" == "gke" ]; then
+    if [ -n "$GCP_CREDENTIALS" ]; then
+        gcloud auth activate-service-account --key-file "$GCP_CREDENTIALS"
+    fi
+    if [ -n "$GCP_REGION" ]; then
+        gcloud config set compute/region "$GCP_REGION"
+    fi
+    if [ -n "$GCP_ZONE" ]; then
+        gcloud config set compute/zone "$GCP_ZONE"
+    fi
+    gcloud container clusters get-credentials "$CLUSTER"
+elif [ "$PROVIDER" == "eks" ]; then
+    :
+fi
+
 if [ -z "$KUBECONTEXT" ]; then
-    KUBECONTEXT=$(kubectl config current-context)
     echo "info: KUBECONTEXT is not set, current context $KUBECONTEXT is used"
+    KUBECONTEXT=$(kubectl config current-context 2>/dev/null) || true
+    if [ -z "$KUBECONTEXT" ]; then
+        echo "error: KUBECONTEXT cannot be detected"
+        exit 1
+    fi
 fi
 
 if [ -z "$SKIP_IMAGE_LOAD" ]; then
@@ -298,7 +320,7 @@ e2e_args=(
     ${ginkgo_args[@]:-}
     /usr/local/bin/e2e.test
     --
-    --provider=skeleton
+    --provider=${PROVIDER}
     --clean-start=true
     --delete-namespace-on-failure=false
     --repo-root=$ROOT
@@ -339,6 +361,16 @@ if [ "$PROVIDER" == "eks" ]; then
     docker_args+=(
         -v $HOME/.aws:/root/.aws
     )
+elif [ "$PROVIDER" == "gke" ]; then
+	e2e_args+=(
+		--gce-project ${GCP_PROJECT}
+		--gce-region ${GCP_REGION}
+		--gce-zone ${GCP_ZONE}
+	)
+	docker_args+=(
+		-v ${GCP_CREDENTIALS}:${GCP_CREDENTIALS}
+		--env GOOGLE_APPLICATION_CREDENTIALS=${GCP_CREDENTIALS}
+	)
 fi
 
 if [ -n "$REPORT_DIR" ]; then

tests/e2e/e2e.go

@@ -35,6 +35,7 @@ import (
 	e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config"
 	utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image"
 	v1 "k8s.io/api/core/v1"
+	storagev1 "k8s.io/api/storage/v1"
 	apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	runtimeutils "k8s.io/apimachinery/pkg/util/runtime"
@@ -43,6 +44,7 @@ import (
 	"k8s.io/component-base/logs"
 	"k8s.io/klog"
 	aggregatorclientset "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset"
+	storageutil "k8s.io/kubernetes/pkg/apis/storage/v1/util"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2elog "k8s.io/kubernetes/test/e2e/framework/log"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
@@ -120,6 +122,39 @@ func setupSuite() {
 		e2elog.Logf("WARNING: Waiting for all daemonsets to be ready failed: %v", err)
 	}
 
+	// By using default storage class in GKE/EKS, network attached storage
+	// which be used and we must clean them later.
+	// We set local-storage class as default for simplicity.
+	// The default storage class of kind is local-path-provisioner which
+	// consumes local storage like local-volume-provisioner.
+	if framework.TestContext.Provider == "gke" || framework.TestContext.Provider == "eks" {
+		defaultSCName := "local-storage"
+		list, err := c.StorageV1().StorageClasses().List(metav1.ListOptions{})
+		framework.ExpectNoError(err)
+		// only one storage class can be marked default
+		// https://kubernetes.io/docs/tasks/administer-cluster/change-default-storage-class/#changing-the-default-storageclass
+		var localStorageSC *storagev1.StorageClass
+		for i, sc := range list.Items {
+			if sc.Name == defaultSCName {
+				localStorageSC = &list.Items[i]
+			} else if storageutil.IsDefaultAnnotation(sc.ObjectMeta) {
+				delete(sc.ObjectMeta.Annotations, storageutil.IsDefaultStorageClassAnnotation)
+				_, err = c.StorageV1().StorageClasses().Update(&sc)
+				framework.ExpectNoError(err)
+			}
+		}
+		if localStorageSC == nil {
+			e2elog.Fail("local-storage storage class not found")
+		}
+		if localStorageSC.Annotations == nil {
+			localStorageSC.Annotations = map[string]string{}
+		}
+		localStorageSC.Annotations[storageutil.IsDefaultStorageClassAnnotation] = "true"
+		e2elog.Logf("Setting %q as the default storage class", localStorageSC.Name)
+		_, err = c.StorageV1().StorageClasses().Update(localStorageSC)
+		framework.ExpectNoError(err)
+	}
+
 	// Log the version of the server and this client.
 	e2elog.Logf("e2e test version: %s", version.Get().GitVersion)