From 75016a516a5a34a4d7f748df15167fb87e97e885 Mon Sep 17 00:00:00 2001 From: Yecheng Fu Date: Thu, 19 Mar 2020 14:47:28 +0800 Subject: [PATCH] add stability e2e group and a basic case --- ci/e2e_kind.groovy | 220 +++++++++++++++++++++ ci/pingcap_tidb_operator_build_kind.groovy | 6 +- hack/lib.sh | 2 +- tests/e2e/e2e.go | 5 + tests/e2e/tidbcluster/serial.go | 5 +- tests/e2e/tidbcluster/stability.go | 189 ++++++++++++++++++ tests/e2e/util/pod/pod.go | 15 +- tests/e2e/util/tidb/tidb.go | 57 ++++++ 8 files changed, 488 insertions(+), 11 deletions(-) create mode 100644 ci/e2e_kind.groovy create mode 100644 tests/e2e/tidbcluster/stability.go create mode 100644 tests/e2e/util/tidb/tidb.go diff --git a/ci/e2e_kind.groovy b/ci/e2e_kind.groovy new file mode 100644 index 0000000000..245b75a3b8 --- /dev/null +++ b/ci/e2e_kind.groovy @@ -0,0 +1,220 @@ +// +// Jenkins pipeline for Kind e2e job. +// +// This script is written in declarative syntax. Refer to +// https://jenkins.io/doc/book/pipeline/syntax/ for more details. +// +// Note that parameters of the job is configured in this script. +// + +import groovy.transform.Field + +@Field +def podYAML = ''' +apiVersion: v1 +kind: Pod +metadata: + labels: + app: tidb-operator-e2e +spec: + containers: + - name: main + image: gcr.io/k8s-testimages/kubekins-e2e:v20200311-1e25827-master + command: + - runner.sh + # Clean containers on TERM signal in root process to avoid cgroup leaking. + # https://github.com/pingcap/tidb-operator/issues/1603#issuecomment-582402196 + - exec + - bash + - -c + - | + function clean() { + echo "info: clean all containers to avoid cgroup leaking" + docker kill $(docker ps -q) || true + docker system prune -af || true + } + trap clean TERM + sleep 1d & wait + # we need privileged mode in order to do docker in docker + securityContext: + privileged: true + env: + - name: DOCKER_IN_DOCKER_ENABLED + value: "true" + resources: + requests: + memory: "8000Mi" + cpu: 8000m + ephemeral-storage: "50Gi" + limits: + memory: "8000Mi" + cpu: 8000m + ephemeral-storage: "50Gi" + # kind needs /lib/modules and cgroups from the host + volumeMounts: + - mountPath: /lib/modules + name: modules + readOnly: true + - mountPath: /sys/fs/cgroup + name: cgroup + # dind expects /var/lib/docker to be volume + - name: docker-root + mountPath: /var/lib/docker + # legacy docker path for cr.io/k8s-testimages/kubekins-e2e + - name: docker-graph + mountPath: /docker-graph + volumes: + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup + type: Directory + - name: docker-root + emptyDir: {} + - name: docker-graph + emptyDir: {} + tolerations: + - effect: NoSchedule + key: tidb-operator + operator: Exists + affinity: + # running on nodes for tidb-operator only + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: ci.pingcap.com + operator: In + values: + - tidb-operator + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: app + operator: In + values: + - tidb-operator-e2e + topologyKey: kubernetes.io/hostname +''' + +// Able to override default values in Jenkins job via environment variables. +if (!env.DEFAULT_GIT_REF) { + env.DEFAULT_GIT_REF = "master" +} + +if (!env.DEFAULT_GINKGO_NODES) { + env.DEFAULT_GINKGO_NODES = "8" +} + +if (!env.DEFAULT_E2E_ARGS) { + env.DEFAULT_E2E_ARGS = "" +} + +if (!env.DEFAULT_DOCKER_IO_MIRROR) { + env.DEFAULT_DOCKER_IO_MIRROR = "" +} + +if (!env.DEFAULT_QUAY_IO_MIRROR) { + env.DEFAULT_QUAY_IO_MIRROR = "" +} + +if (!env.DEFAULT_GCR_IO_MIRROR) { + env.DEFAULT_GCR_IO_MIRROR = "" +} + +pipeline { + agent { + kubernetes { + yaml podYAML + defaultContainer "main" + customWorkspace "/home/jenkins/agent/workspace/go/src/github.com/pingcap/tidb-operator" + } + } + + options { + timeout(time: 3, unit: 'HOURS') + } + + parameters { + string(name: 'GIT_URL', defaultValue: 'git@github.com:pingcap/tidb-operator.git', description: 'git repo url') + string(name: 'GIT_REF', defaultValue: env.DEFAULT_GIT_REF, description: 'git ref spec to checkout, e.g. master, release-1.1') + string(name: 'PR_ID', defaultValue: '', description: 'pull request ID, this will override GIT_REF if set, e.g. 1889') + string(name: 'GINKGO_NODES', defaultValue: env.DEFAULT_GINKGO_NODES, description: 'the number of ginkgo nodes') + string(name: 'E2E_ARGS', defaultValue: env.DEFAULT_E2E_ARGS, description: "e2e args, e.g. --ginkgo.focus='\\[Stability\\]'") + string(name: 'DOCKER_IO_MIRROR', defaultValue: env.DEFAULT_DOCKER_IO_MIRROR, description: "docker mirror for docker.io") + string(name: 'QUAY_IO_MIRROR', defaultValue: env.DEFAULT_QUAY_IO_MIRROR, description: "mirror for quay.io") + string(name: 'GCR_IO_MIRROR', defaultValue: env.DEFAULT_GCR_IO_MIRROR, description: "mirror for gcr.io") + } + + environment { + GIT_REF = '' + ARTIFACTS = "${env.WORKSPACE}/artifacts" + } + + stages { + stage("Prepare") { + steps { + // The declarative model for Jenkins Pipelines has a restricted + // subset of syntax that it allows in the stage blocks. We use + // script step to bypass the restriction. + // https://jenkins.io/doc/book/pipeline/syntax/#script + script { + GIT_REF = params.GIT_REF + if (params.PR_ID != "") { + GIT_REF = "refs/remotes/origin/pr/${params.PR_ID}/head" + } + } + echo "env.NODE_NAME: ${env.NODE_NAME}" + echo "env.WORKSPACE: ${env.WORKSPACE}" + echo "GIT_REF: ${GIT_REF}" + echo "ARTIFACTS: ${ARTIFACTS}" + } + } + + stage("Checkout") { + steps { + checkout scm: [ + $class: 'GitSCM', + branches: [[name: GIT_REF]], + userRemoteConfigs: [[ + credentialsId: 'github-sre-bot-ssh', + refspec: '+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*', + url: "${params.GIT_URL}", + ]] + ] + } + } + + stage("Run") { + steps { + sh """ + #!/bin/bash + export GINKGO_NODES=${params.GINKGO_NODES} + export REPORT_DIR=${ARTIFACTS} + export DOCKER_IO_MIRROR=${params.DOCKER_IO_MIRROR} + export QUAY_IO_MIRROR=${params.QUAY_IO_MIRROR} + export GCR_IO_MIRROR=${params.GCR_IO_MIRROR} + echo "info: begin to run e2e" + ./hack/e2e.sh -- ${params.E2E_ARGS} + """ + } + } + } + + post { + always { + dir(ARTIFACTS) { + archiveArtifacts artifacts: "**", allowEmptyArchive: true + junit testResults: "*.xml", allowEmptyResults: true + } + } + } +} + +// vim: et sw=4 ts=4 diff --git a/ci/pingcap_tidb_operator_build_kind.groovy b/ci/pingcap_tidb_operator_build_kind.groovy index aaa97ed6ff..70ab0486bb 100644 --- a/ci/pingcap_tidb_operator_build_kind.groovy +++ b/ci/pingcap_tidb_operator_build_kind.groovy @@ -238,13 +238,13 @@ def call(BUILD_BRANCH, CREDENTIALS_ID, CODECOV_CREDENTIALS_ID) { def MIRRORS = "DOCKER_IO_MIRROR=http://172.16.4.143:5000 QUAY_IO_MIRROR=http://172.16.4.143:5001" def builds = [:] builds["E2E v1.12.10"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]'", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_ ./hack/e2e.sh -- --preload-images", artifacts) } builds["E2E v1.12.10 AdvancedStatefulSet"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --ginkgo.skip='\\[Serial\\]' --operator-features AdvancedStatefulSet=true", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-advanced-statefulset IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_advanced_statefulset ./hack/e2e.sh -- --preload-images --operator-features AdvancedStatefulSet=true", artifacts) } builds["E2E v1.17.0"] = { - build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images --ginkgo.skip='\\[Serial\\]'", artifacts) + build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.17 IMAGE_TAG=${GITHASH} SKIP_BUILD=y GINKGO_NODES=6 KUBE_VERSION=v1.17.0 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.17.0_ ./hack/e2e.sh -- -preload-images", artifacts) } builds["E2E v1.12.10 Serial"] = { build("${MIRRORS} RUNNER_SUITE_NAME=e2e-v1.12-serial IMAGE_TAG=${GITHASH} SKIP_BUILD=y KUBE_VERSION=v1.12.10 REPORT_DIR=\$(pwd)/artifacts REPORT_PREFIX=v1.12.10_serial_ ./hack/e2e.sh -- --preload-images --ginkgo.focus='\\[Serial\\]' --install-operator=false", artifacts) diff --git a/hack/lib.sh b/hack/lib.sh index 3330541ad6..3e46051598 100644 --- a/hack/lib.sh +++ b/hack/lib.sh @@ -154,7 +154,7 @@ function hack::wait_for_success() { } # -# Concatenates the elements with an separator between them. +# Concatenates the elements with a separator between them. # # Usage: hack::join ',' a b c # diff --git a/tests/e2e/e2e.go b/tests/e2e/e2e.go index a7058225aa..35831e8315 100644 --- a/tests/e2e/e2e.go +++ b/tests/e2e/e2e.go @@ -274,6 +274,11 @@ func RunE2ETests(t *testing.T) { gomega.RegisterFailHandler(e2elog.Fail) + // Disable serial and stability tests by default unless they are explicitly requested. + if config.GinkgoConfig.FocusString == "" && config.GinkgoConfig.SkipString == "" { + config.GinkgoConfig.SkipString = `\[Stability\]|\[Serial\]` + } + // Run tests through the Ginkgo runner with output to console + JUnit for Jenkins var r []ginkgo.Reporter if framework.TestContext.ReportDir != "" { diff --git a/tests/e2e/tidbcluster/serial.go b/tests/e2e/tidbcluster/serial.go index 7bfffd8fa0..f476791b54 100644 --- a/tests/e2e/tidbcluster/serial.go +++ b/tests/e2e/tidbcluster/serial.go @@ -62,6 +62,7 @@ func mustToString(set sets.Int32) string { return string(b) } +// Serial specs describe tests which cannot run in parallel. var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { f := framework.NewDefaultFramework("serial") @@ -490,8 +491,8 @@ var _ = ginkgo.Describe("[tidb-operator][Serial]", func() { }) framework.ExpectNoError(err) - ginkgo.By("Make sure pods are not affected") - err = utilpod.WaitForPodsAreNotAffected(c, podListBeforeUpgrade.Items, time.Minute*3) + ginkgo.By("Make sure pods are not changed") + err = utilpod.WaitForPodsAreChanged(c, podListBeforeUpgrade.Items, time.Minute*3) framework.ExpectEqual(err, wait.ErrWaitTimeout, "Pods was not affeteced after the operator is upgraded") }) diff --git a/tests/e2e/tidbcluster/stability.go b/tests/e2e/tidbcluster/stability.go new file mode 100644 index 0000000000..88d2d8fb05 --- /dev/null +++ b/tests/e2e/tidbcluster/stability.go @@ -0,0 +1,189 @@ +// Copyright 2019 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tidbcluster + +import ( + "context" + "fmt" + _ "net/http/pprof" + "time" + + "github.com/onsi/ginkgo" + asclientset "github.com/pingcap/advanced-statefulset/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/client/clientset/versioned" + "github.com/pingcap/tidb-operator/pkg/label" + "github.com/pingcap/tidb-operator/pkg/scheme" + "github.com/pingcap/tidb-operator/tests" + e2econfig "github.com/pingcap/tidb-operator/tests/e2e/config" + utilimage "github.com/pingcap/tidb-operator/tests/e2e/util/image" + utilpod "github.com/pingcap/tidb-operator/tests/e2e/util/pod" + "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" + utiltidb "github.com/pingcap/tidb-operator/tests/e2e/util/tidb" + "github.com/pingcap/tidb-operator/tests/pkg/fixture" + v1 "k8s.io/api/core/v1" + apiextensionsclientset "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/util/wait" + clientset "k8s.io/client-go/kubernetes" + restclient "k8s.io/client-go/rest" + aggregatorclient "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset" + "k8s.io/kubernetes/test/e2e/framework" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// Stability specs describe tests which involve disruptive operations, e.g. +// stop kubelet, kill nodes, empty pd/tikv data. +// Like serial tests, they cannot run in parallel too. +var _ = ginkgo.Describe("[tidb-operator][Stability]", func() { + f := framework.NewDefaultFramework("stability") + + var ns string + var c clientset.Interface + var cli versioned.Interface + var asCli asclientset.Interface + var aggrCli aggregatorclient.Interface + var apiExtCli apiextensionsclientset.Interface + var cfg *tests.Config + var config *restclient.Config + var fw portforward.PortForward + var fwCancel context.CancelFunc + + ginkgo.BeforeEach(func() { + ns = f.Namespace.Name + c = f.ClientSet + var err error + config, err = framework.LoadConfig() + framework.ExpectNoError(err, "failed to load config") + cli, err = versioned.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + asCli, err = asclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + aggrCli, err = aggregatorclient.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + apiExtCli, err = apiextensionsclientset.NewForConfig(config) + framework.ExpectNoError(err, "failed to create clientset") + clientRawConfig, err := e2econfig.LoadClientRawConfig() + framework.ExpectNoError(err, "failed to load raw config") + ctx, cancel := context.WithCancel(context.Background()) + fw, err = portforward.NewPortForwarder(ctx, e2econfig.NewSimpleRESTClientGetter(clientRawConfig)) + framework.ExpectNoError(err, "failed to create port forwarder") + fwCancel = cancel + cfg = e2econfig.TestConfig + }) + + ginkgo.AfterEach(func() { + if fwCancel != nil { + fwCancel() + } + }) + + // TODO generate more contexts for different operator values + ginkgo.Context("operator with default values", func() { + var ocfg *tests.OperatorConfig + var oa tests.OperatorActions + var genericCli client.Client + + ginkgo.BeforeEach(func() { + ocfg = &tests.OperatorConfig{ + Namespace: "pingcap", + ReleaseName: "operator", + Image: cfg.OperatorImage, + Tag: cfg.OperatorTag, + LogLevel: "4", + TestMode: true, + } + oa = tests.NewOperatorActions(cli, c, asCli, aggrCli, apiExtCli, tests.DefaultPollInterval, ocfg, e2econfig.TestConfig, nil, fw, f) + ginkgo.By("Installing CRDs") + oa.CleanCRDOrDie() + oa.InstallCRDOrDie(ocfg) + ginkgo.By("Installing tidb-operator") + oa.CleanOperatorOrDie(ocfg) + oa.DeployOperatorOrDie(ocfg) + var err error + genericCli, err = client.New(config, client.Options{Scheme: scheme.Scheme}) + framework.ExpectNoError(err, "failed to create clientset") + }) + + ginkgo.AfterEach(func() { + ginkgo.By("Uninstall tidb-operator") + oa.CleanOperatorOrDie(ocfg) + ginkgo.By("Uninstalling CRDs") + oa.CleanCRDOrDie() + }) + + testCases := []struct { + name string + fn func() + }{ + { + name: "tidb-operator does not exist", + fn: func() { + ginkgo.By("Uninstall tidb-operator") + oa.CleanOperatorOrDie(ocfg) + }, + }, + } + + for _, test := range testCases { + ginkgo.It("tidb cluster should not be affected while "+test.name, func() { + clusterName := "test" + tc := fixture.GetTidbCluster(ns, clusterName, utilimage.TiDBV3Version) + err := genericCli.Create(context.TODO(), tc) + framework.ExpectNoError(err) + err = oa.WaitForTidbClusterReady(tc, 30*time.Minute, 15*time.Second) + framework.ExpectNoError(err) + + test.fn() + + ginkgo.By("Check tidb cluster is not affected") + listOptions := metav1.ListOptions{ + LabelSelector: labels.SelectorFromSet(label.New().Instance(clusterName).Labels()).String(), + } + podList, err := c.CoreV1().Pods(ns).List(listOptions) + framework.ExpectNoError(err) + err = wait.PollImmediate(time.Second*30, time.Minute*5, func() (bool, error) { + var ok bool + var err error + framework.Logf("check whether pods of cluster %q are changed", clusterName) + ok, err = utilpod.PodsAreChanged(c, podList.Items)() + if ok || err != nil { + // pod changed or some error happened + return true, err + } + framework.Logf("check whether pods of cluster %q are running", clusterName) + newPodList, err := c.CoreV1().Pods(ns).List(listOptions) + if err != nil { + return false, err + } + for _, pod := range newPodList.Items { + if pod.Status.Phase != v1.PodRunning { + return false, fmt.Errorf("pod %s/%s is not running", pod.Namespace, pod.Name) + } + } + framework.Logf("check whehter tidb cluster %q is connectable", clusterName) + ok, err = utiltidb.TiDBIsConnectable(fw, ns, clusterName, "root", "")() + if !ok || err != nil { + // not connectable or some error happened + return true, err + } + return false, nil + }) + framework.ExpectEqual(err, wait.ErrWaitTimeout, "TiDB cluster is not affeteced") + }) + } + + }) + +}) diff --git a/tests/e2e/util/pod/pod.go b/tests/e2e/util/pod/pod.go index b0a95ab8b2..f84abd0ce7 100644 --- a/tests/e2e/util/pod/pod.go +++ b/tests/e2e/util/pod/pod.go @@ -26,10 +26,9 @@ import ( testutils "k8s.io/kubernetes/test/utils" ) -// WaitForPodsAreNotAffected waits for given pods are not affected. -// It returns wait.ErrWaitTimeout if the given pods are not affected in specified timeout. -func WaitForPodsAreNotAffected(c kubernetes.Interface, pods []v1.Pod, timeout time.Duration) error { - return wait.PollImmediate(time.Second*5, timeout, func() (bool, error) { +// PodsAreChanged checks the given pods are changed or not (recreate, update). +func PodsAreChanged(c kubernetes.Interface, pods []v1.Pod) wait.ConditionFunc { + return func() (bool, error) { for _, pod := range pods { podNew, err := c.CoreV1().Pods(pod.Namespace).Get(pod.Name, metav1.GetOptions{}) if err != nil { @@ -46,5 +45,11 @@ func WaitForPodsAreNotAffected(c kubernetes.Interface, pods []v1.Pod, timeout ti } } return false, nil - }) + } +} + +// WaitForPodsAreChanged waits for given pods are changed. +// It returns wait.ErrWaitTimeout if the given pods are not changed in specified timeout. +func WaitForPodsAreChanged(c kubernetes.Interface, pods []v1.Pod, timeout time.Duration) error { + return wait.PollImmediate(time.Second*5, timeout, PodsAreChanged(c, pods)) } diff --git a/tests/e2e/util/tidb/tidb.go b/tests/e2e/util/tidb/tidb.go new file mode 100644 index 0000000000..aba1656dad --- /dev/null +++ b/tests/e2e/util/tidb/tidb.go @@ -0,0 +1,57 @@ +// Copyright 2020 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// See the License for the specific language governing permissions and +// limitations under the License. + +package tidb + +import ( + "context" + "database/sql" + "fmt" + + // To register MySQL driver + _ "github.com/go-sql-driver/mysql" + "github.com/pingcap/tidb-operator/pkg/controller" + "github.com/pingcap/tidb-operator/tests/e2e/util/portforward" + "k8s.io/apimachinery/pkg/util/wait" +) + +var dummyCancel = func() {} + +// GetTiDBDSN returns a DSN to use +func GetTiDBDSN(fw portforward.PortForward, ns, tc, user, password, database string) (string, context.CancelFunc, error) { + localHost, localPort, cancel, err := portforward.ForwardOnePort(fw, ns, fmt.Sprintf("svc/%s", controller.TiDBMemberName(tc)), 4000) + if err != nil { + return "", dummyCancel, err + } + return fmt.Sprintf("%s:%s@(%s:%d)/%s?charset=utf8", user, password, localHost, localPort, database), cancel, nil +} + +// TiDBIsConnectable checks whether the tidb cluster is connectable. +func TiDBIsConnectable(fw portforward.PortForward, ns, tc, user, password string) wait.ConditionFunc { + return func() (bool, error) { + var db *sql.DB + dsn, cancel, err := GetTiDBDSN(fw, ns, tc, "root", password, "test") + if err != nil { + return false, err + } + defer cancel() + if db, err = sql.Open("mysql", dsn); err != nil { + return false, err + } + defer db.Close() + if err := db.Ping(); err != nil { + return false, err + } + return true, nil + } +}