Cherry pick pr 1102: add e2e scripts and simplify e2e jenkins file (p…

…ingcap#1198)

* add e2e scripts and simplify e2e jenkins file (pingcap#1102)

* e2e scripts

* fix

* fix

* fix

* copy and modify

* check e2e status

* Update hack/run-e2e.sh

Co-Authored-By: Tennix <tennix@users.noreply.github.com>

* Update hack/run-e2e.sh

Co-Authored-By: Tennix <tennix@users.noreply.github.com>

* simplify

* Update hack/e2e.sh

Co-Authored-By: Tennix <tennix@users.noreply.github.com>

* retry on non-fatal errors

* fix

* fix

* fix image repalces

* fix

* wait for the secret for sa is created

* vendor kubectl binary

* setup go

* fix

Makefile

@@ -17,6 +17,7 @@ GOENV  := GO15VENDOREXPERIMENT="1" GO111MODULE=on CGO_ENABLED=0 GOOS=$(GOOS) GOA
 GO     := $(GOENV) go build -trimpath
 GOTEST := CGO_ENABLED=0 GO111MODULE=on go test -v -cover
 
+IMAGE_TAG ?= latest
 PACKAGE_LIST := go list ./... | grep -vE "pkg/client" | grep -vE "zz_generated"
 PACKAGE_DIRECTORIES := $(PACKAGE_LIST) | sed 's|github.com/pingcap/tidb-operator/||'
 FILES := $$(find $$($(PACKAGE_DIRECTORIES)) -name "*.go")
@@ -26,10 +27,15 @@ TEST_COVER_PACKAGES:=go list ./pkg/... | grep -vE "pkg/client" | grep -vE "pkg/t
 default: build
 
 docker-push: docker
-	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator:${IMAGE_TAG}"
 
+ifeq ($(NO_BUILD),y)
+docker:
+	@echo "NO_BUILD=y, skip build for $@"
+else
 docker: build
-	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:latest" images/tidb-operator
+endif
+	docker build --tag "${DOCKER_REGISTRY}/pingcap/tidb-operator:${IMAGE_TAG}" images/tidb-operator
 
 build: controller-manager scheduler discovery admission-controller
 
@@ -50,9 +56,14 @@ e2e-setup:
 	@GO111MODULE=on CGO_ENABLED=0 go get github.com/onsi/ginkgo@v1.6.0
 
 e2e-docker-push: e2e-docker
-	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:${IMAGE_TAG}"
 
+ifeq ($(NO_BUILD),y)
+e2e-docker:
+	@echo "NO_BUILD=y, skip build for $@"
+else
 e2e-docker: e2e-build
+endif
 	[ -d tests/images/e2e/tidb-operator ] && rm -r tests/images/e2e/tidb-operator || true
 	[ -d tests/images/e2e/tidb-cluster ] && rm -r tests/images/e2e/tidb-cluster || true
 	[ -d tests/images/e2e/tidb-backup ] && rm -r tests/images/e2e/tidb-backup || true
@@ -61,19 +72,22 @@ e2e-docker: e2e-build
 	cp -r charts/tidb-cluster tests/images/e2e
 	cp -r charts/tidb-backup tests/images/e2e
 	cp -r manifests tests/images/e2e
-	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:latest" tests/images/e2e
+	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-e2e:${IMAGE_TAG}" tests/images/e2e
 
 e2e-build: e2e-setup
 	$(GO) -ldflags '$(LDFLAGS)' -o tests/images/e2e/bin/e2e tests/cmd/e2e/main.go
 
+e2e:
+	./hack/e2e.sh
+
 stability-test-build:
 	$(GO) -ldflags '$(LDFLAGS)' -o tests/images/stability-test/bin/stability-test tests/cmd/stability/*.go
 
 stability-test-docker: stability-test-build
-	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-stability-test:latest" tests/images/stability-test
+	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-operator-stability-test:${IMAGE_TAG}" tests/images/stability-test
 
 stability-test-push: stability-test-docker
-	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-stability-test:latest"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-operator-stability-test:${IMAGE_TAG}"
 
 fault-trigger:
 	$(GO) -ldflags '$(LDFLAGS)' -o tests/images/fault-trigger/bin/fault-trigger tests/cmd/fault-trigger/*.go
@@ -139,16 +153,16 @@ cli:
 	$(GO) -ldflags '$(LDFLAGS)' -o tkctl cmd/tkctl/main.go
 
 debug-docker-push: debug-build-docker
-	docker push "${DOCKER_REGISTRY}/pingcap/debug-launcher:latest"
-	docker push "${DOCKER_REGISTRY}/pingcap/tidb-control:latest"
-	docker push "${DOCKER_REGISTRY}/pingcap/tidb-debug:latest"
+	docker push "${DOCKER_REGISTRY}/pingcap/debug-launcher:${IMAGE_TAG}"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-control:${IMAGE_TAG}"
+	docker push "${DOCKER_REGISTRY}/pingcap/tidb-debug:${IMAGE_TAG}"
 
 debug-build-docker: debug-build
-	docker build -t "${DOCKER_REGISTRY}/pingcap/debug-launcher:latest" misc/images/debug-launcher
-	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-control:latest" misc/images/tidb-control
-	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-debug:latest" misc/images/tidb-debug
+	docker build -t "${DOCKER_REGISTRY}/pingcap/debug-launcher:${IMAGE_TAG}" misc/images/debug-launcher
+	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-control:${IMAGE_TAG}" misc/images/tidb-control
+	docker build -t "${DOCKER_REGISTRY}/pingcap/tidb-debug:${IMAGE_TAG}" misc/images/tidb-debug
 
 debug-build:
 	$(GO) -ldflags '$(LDFLAGS)' -o misc/images/debug-launcher/bin/debug-launcher misc/cmd/debug-launcher/main.go
 
-.PHONY: check check-setup check-all build e2e-build debug-build cli
+.PHONY: check check-setup check-all build e2e-build debug-build cli e2e

ci/pingcap_tidb_operator_build_kind.groovy

@@ -1,17 +1,6 @@
-def replace(file, operatorImage, e2eImage, testApiserverImage) {
-	def SRC_E2E_FILE_CONTENT = readFile file: file
-	def DST_E2E_FILE_CONTENT = SRC_E2E_FILE_CONTENT.replaceAll("- --operator-image=.*", "- --operator-image=${operatorImage}")
-	DST_E2E_FILE_CONTENT = DST_E2E_FILE_CONTENT.replaceAll("image:.*", "image: ${e2eImage}")
-	DST_E2E_FILE_CONTENT = DST_E2E_FILE_CONTENT.replaceAll("- --test-apiserver-image=.*", "- --test-apiserver-image=${testApiserverImage}")
-	DST_E2E_FILE_CONTENT = DST_E2E_FILE_CONTENT.replaceAll("imagePullPolicy: Always", "imagePullPolicy: IfNotPresent")
-	writeFile file: file, text: "${DST_E2E_FILE_CONTENT}"
-}
-
-def replace_wh(file, operatorImage) {
-	def SRC_WH_FILE_CONTENT = readFile file: file
-	def DST_WH_FILE_CONTENT = SRC_WH_FILE_CONTENT.replaceAll("image:.*", "image: ${operatorImage}")
-	writeFile file: file, text: "${DST_WH_FILE_CONTENT}"
-}
+//
+// E2E Jenkins file.
+//
 
 def getChangeLogText() {
 	def changeLogText = ""
@@ -48,54 +37,43 @@ def call(BUILD_BRANCH, CREDENTIALS_ID) {
 							sudo yum install -y mercurial
 						fi
 						hg --version
+						make check-setup
+						make check
+						make test
 						make
-						rm -rf tests/images/test-apiserver/bin
 						make e2e-build
 						"""
 					}
 				}
-				stash excludes: "${PROJECT_DIR}/vendor/**", includes: "${PROJECT_DIR}/**", name: "tidb-operator"
+				stash excludes: "${PROJECT_DIR}/vendor/**,${PROJECT_DIR}/deploy/**", includes: "${PROJECT_DIR}/**", name: "tidb-operator"
 			}
 		}
 
 		node('k8s-kind') {
-			def WORKSPACE = pwd()
-			def E2E_IMAGE = "localhost:5000/pingcap/tidb-operator-e2e:${GITHASH.take(7)}"
-			def IMAGE_TAG = "localhost:5000/pingcap/tidb-operator:${GITHASH.take(7)}"
-			def APISERVER_IMAGE = "localhost:5000/pingcap/test-apiserver:${GITHASH.take(7)}"
 			deleteDir()
 			unstash 'tidb-operator'
 
 			dir("${PROJECT_DIR}"){
-				stage('build tidb-operator image'){
-					sh """
-					docker build -t ${IMAGE_TAG} images/tidb-operator
-					"""
-				}
-
-				stage('start prepare runtime environment'){
-					def wh_yaml = "manifests/webhook.yaml"
-					replace_wh(wh_yaml, IMAGE_TAG)
-					sh """
-					cp -r charts/tidb-operator tests/images/e2e
-					cp -r charts/tidb-cluster tests/images/e2e
-					cp -r charts/tidb-backup tests/images/e2e
-					cp -r manifests tests/images/e2e
-					docker build -t ${E2E_IMAGE} tests/images/e2e
-					if [ -f tests/images/test-apiserver/bin/tidb-apiserver ]; then docker build -t ${APISERVER_IMAGE} tests/images/test-apiserver; fi
-					"""
-				}
-
 				stage('start run operator e2e test'){
-					def operator_yaml = "tests/manifests/e2e/e2e.yaml"
-					replace(operator_yaml, IMAGE_TAG, E2E_IMAGE, APISERVER_IMAGE)
 					ansiColor('xterm') {
-					def ns = "tidb-operator-e2e"
 					sh """#/usr/bin/env bash
+					echo "info: setup go"
+					export PATH=\$PATH:/usr/local/go/bin
+					echo "==== Start Build Environment ===="
 					echo "BASH VERSION: \$BASH_VERSION"
-					tidbOperatorImage="${IMAGE_TAG}"
-					tidbOperatorE2EImage="${E2E_IMAGE}"
-					tidbOperatorApiServerImage="${APISERVER_IMAGE}"
+					echo "PATH: \$PATH"
+					echo "PROJECT_DIR: ${PROJECT_DIR}"
+					if ! command -v go &>/dev/null; then
+						echo "error: go is required"
+						exit 1
+					fi
+					go version
+					if ! command -v docker &>/dev/null; then
+						echo "error: docker is required"
+						exit 1
+					fi
+					docker version
+					echo "==== End Build Environment ===="
 					declare -A clusters
 					clusters['kind']='127.0.0.1:5000'
 					clusters['kind2']='127.0.0.2:5000'
@@ -135,78 +113,8 @@ def call(BUILD_BRANCH, CREDENTIALS_ID) {
 					done
 					trap " echo '############ end of e2e test running on the cluster: \$clusterName ############'; rm -f /k8s-locks/\$clusterName " INT TERM EXIT
 
-					tidbOperatorImage=\${tidbOperatorImage/localhost:5000/\${clusters[\$clusterName]}}
-					tidbOperatorE2EImage=\${tidbOperatorE2EImage/localhost:5000/\${clusters[\$clusterName]}}
-					tidbOperatorApiServerImage=\${tidbOperatorApiServerImage/localhost:5000/\${clusters[\$clusterName]}}
-					echo "pushing tidb-operator image ${IMAGE_TAG} via \$tidbOperatorImage"
-					docker tag ${IMAGE_TAG} \$tidbOperatorImage
-					docker push \$tidbOperatorImage
-					docker rmi -f ${IMAGE_TAG}
-					docker rmi -f \$tidbOperatorImage
-					echo "pushing tidb-operator e2e image ${E2E_IMAGE} via \$tidbOperatorE2EImage"
-					docker tag ${E2E_IMAGE} \$tidbOperatorE2EImage
-					docker push \$tidbOperatorE2EImage
-					docker rmi -f ${E2E_IMAGE}
-					docker rmi -f \$tidbOperatorE2EImage
-					echo "pushing apiserver e2e image ${APISERVER_IMAGE} via \$tidbOperatorApiServerImage"
-					if [ -f tests/images/test-apiserver/bin/tidb-apiserver ]; then
-						docker tag ${APISERVER_IMAGE} \$tidbOperatorApiServerImage
-						docker push \$tidbOperatorApiServerImage
-						docker rmi -f ${APISERVER_IMAGE}
-						docker rmi -f \$tidbOperatorApiServerImage
-					fi
-					export KUBECONFIG=`/root/go/bin/kind get kubeconfig-path --name="\$clusterName"`
-					elapseTime=0
-					period=5
-					threshold=300
-
-					kubectl create ns ${ns} || true
-					kubectl delete validatingwebhookconfiguration --all
-					sleep 5
-					kubectl delete -f ${operator_yaml} --ignore-not-found
-					kubectl wait --for=delete -n ${ns} pod/tidb-operator-e2e || true
-					# Create sa first and wait for the controller to create the secret for this sa
-					# This avoids `No API token found for service account " tidb-operator-e2e"`
-					# TODO better way to work around this issue
-					kubectl -n tidb-operator-e2e create sa tidb-operator-e2e
-					kubectl apply -f ${operator_yaml}
-
-					while true
-					do
-						sleep \$period
-						elapseTime=\$(( elapseTime+\$period ))
-						kubectl get po/tidb-operator-e2e -n ${ns} 2>/dev/null || continue
-						kubectl get po/tidb-operator-e2e -n ${ns}|grep Running && break || true
-						if [[ \$elapseTime -gt \$threshold ]]
-						then
-							echo "wait e2e pod timeout, elapseTime: \$elapseTime"
-							exit 1
-						fi
-					done
-
-					execType=0
-					while true
-					do
-						if [[ \$execType -eq 0 ]]
-						then
-							kubectl logs -f tidb-operator-e2e -n ${ns}
-						else
-							execType=0
-							kubectl logs --tail 1 -f tidb-operator-e2e -n ${ns}
-						fi
-						## verify that the pod status is exit normally
-						status=`kubectl get po tidb-operator-e2e -n ${ns} -ojsonpath='{.status.phase}'`
-						if [[ \$status == Succeeded ]]
-						then
-							exit 0
-						elif [[ \$status == Failed ]]
-						then
-							exit 1
-						fi
-						execType=1
-					done
-					trap - INT TERM ERR
-					rm -f /k8s-locks/\$clusterName
+                    export KUBECONFIG=`/root/go/bin/kind get kubeconfig-path --name="\$clusterName"`
+                    DOCKER_REGISTRY=\${clusters[\$clusterName]} IMAGE_TAG=${GITHASH} SKIP_BUILD=y make e2e
 					"""
 					}
 				}
@@ -258,3 +166,5 @@ def call(BUILD_BRANCH, CREDENTIALS_ID) {
 	}
 }
 return this
+
+// vim: noet

deploy/modules/gcp/tidb-operator/versions.tf

@@ -0,0 +1,11 @@
+
+terraform {
+  required_version = ">= 0.12"
+  required_providers {
+    google      = "~> 2.16"
+    google-beta = "~> 2.16"
+    external    = "~> 1.2"
+    helm        = "~> 0.10"
+    null        = "~> 2.1"
+  }
+}

hack/e2e.sh

@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# E2E entrypoint script.
+#
+
+set -o errexit
+set -o nounset
+set -o pipefail
+
+ROOT=$(unset CDPATH && cd $(dirname "${BASH_SOURCE[0]}")/.. && pwd)
+cd $ROOT
+
+function usage() {
+    cat <<'EOF'
+This script is entrypoint to run e2e tests.
+
+Usage: hack/e2e.sh [-h] -- [extra test args]
+
+    -h      show this message and exit
+
+Environments:
+
+	DOCKER_REGISTRY		image docker registry
+	IMAGE_TAG			image tag
+	SKIP_BUILD 			skip building binaries
+	SKIP_IMAGE_BUILD	skip build and push images
+EOF
+
+}
+
+while getopts "h?" opt; do
+    case "$opt" in
+    h|\?)
+        usage
+        exit 0
+        ;;  
+    esac
+done
+
+DOCKER_REGISTRY=${DOCKER_REGISTRY:-localhost:5000}
+IMAGE_TAG=${IMAGE_TAG:-latest}
+SKIP_BUILD=${SKIP_BUILD:-}
+SKIP_IMAGE_BUILD=${SKIP_IMAGE_BUILD:-}
+
+echo "DOCKER_REGISTRY: $DOCKER_REGISTRY"
+echo "IMAGE_TAG: $IMAGE_TAG"
+echo "SKIP_BUILD: $SKIP_BUILD"
+echo "SKIP_IMAGE_BUILD: $SKIP_IMAGE_BUILD"
+
+if [ -n "$SKIP_BUILD" ]; then
+    echo "info: skip building images"
+    export NO_BUILD=y
+fi
+
+if [ -n "$SKIP_IMAGE_BUILD" ]; then
+    echo "info: skip building and pushing images"
+else
+    DOCKER_REGISTRY=$DOCKER_REGISTRY IMAGE_TAG=$IMAGE_TAG make docker-push
+    DOCKER_REGISTRY=$DOCKER_REGISTRY IMAGE_TAG=$IMAGE_TAG make e2e-docker-push
+fi
+
+# in kind cluster, we must use local registry
+# TODO: find a better way
+export TIDB_OPERATOR_IMAGE=localhost:5000/pingcap/tidb-operator:${IMAGE_TAG}
+export E2E_IMAGE=localhost:5000/pingcap/tidb-operator-e2e:${IMAGE_TAG}
+export TEST_APISERVER_IMAGE=localhost:5000/pingcap/test-apiserver:${IMAGE_TAG}
+
+hack/run-e2e.sh "$@"