Skip to content

Commit

Permalink
Remove resign ddl owner logic for tidb-servers
Browse files Browse the repository at this point in the history
Signed-off-by: Aylei <rayingecho@gmail.com>
  • Loading branch information
aylei committed Nov 26, 2019
1 parent f67ed33 commit 5e306b2
Show file tree
Hide file tree
Showing 4 changed files with 6 additions and 141 deletions.
55 changes: 4 additions & 51 deletions pkg/controller/tidb_control.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,6 @@ type dbInfo struct {
type TiDBControlInterface interface {
// GetHealth returns tidb's health info
GetHealth(tc *v1alpha1.TidbCluster) map[string]bool
// ResignDDLOwner resigns the ddl owner of tidb, if the tidb node is not a ddl owner returns (true,nil),else returns (false,err)
ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error)
// Get TIDB info return tidb's dbInfo
GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error)
// GetSettings return the TiDB instance settings
Expand Down Expand Up @@ -98,35 +96,6 @@ func (tdc *defaultTiDBControl) GetHealth(tc *v1alpha1.TidbCluster) map[string]bo
return result
}

func (tdc *defaultTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) {
tcName := tc.GetName()
ns := tc.GetNamespace()
scheme := tc.Scheme()
if err := tdc.useTLSHTTPClient(tc.Spec.EnableTLSCluster); err != nil {
return false, err
}

hostName := fmt.Sprintf("%s-%d", TiDBMemberName(tcName), ordinal)
url := fmt.Sprintf("%s://%s.%s.%s:10080/ddl/owner/resign", scheme, hostName, TiDBPeerMemberName(tcName), ns)
req, err := http.NewRequest("POST", url, nil)
if err != nil {
return false, err
}
res, err := tdc.httpClient.Do(req)
if err != nil {
return false, err
}
defer httputil.DeferClose(res.Body)
if res.StatusCode == http.StatusOK {
return false, nil
}
err2 := httputil.ReadErrorBody(res.Body)
if err2.Error() == NotDDLOwnerError {
return true, nil
}
return false, err2
}

func (tdc *defaultTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) {
tcName := tc.GetName()
ns := tc.GetNamespace()
Expand Down Expand Up @@ -217,12 +186,10 @@ func (tdc *defaultTiDBControl) getBodyOK(apiURL string) ([]byte, error) {

// FakeTiDBControl is a fake implementation of TiDBControlInterface.
type FakeTiDBControl struct {
healthInfo map[string]bool
resignDDLOwnerError error
notDDLOwner bool
tidbInfo *dbInfo
getInfoError error
tidbConfig *config.Config
healthInfo map[string]bool
tidbInfo *dbInfo
getInfoError error
tidbConfig *config.Config
}

// NewFakeTiDBControl returns a FakeTiDBControl instance
Expand All @@ -235,24 +202,10 @@ func (ftd *FakeTiDBControl) SetHealth(healthInfo map[string]bool) {
ftd.healthInfo = healthInfo
}

// NotDDLOwner sets whether the tidb is the ddl owner
func (ftd *FakeTiDBControl) NotDDLOwner(notDDLOwner bool) {
ftd.notDDLOwner = notDDLOwner
}

// SetResignDDLOwner sets error of resign ddl owner for FakeTiDBControl
func (ftd *FakeTiDBControl) SetResignDDLOwnerError(err error) {
ftd.resignDDLOwnerError = err
}

func (ftd *FakeTiDBControl) GetHealth(_ *v1alpha1.TidbCluster) map[string]bool {
return ftd.healthInfo
}

func (ftd *FakeTiDBControl) ResignDDLOwner(tc *v1alpha1.TidbCluster, ordinal int32) (bool, error) {
return ftd.notDDLOwner, ftd.resignDDLOwnerError
}

func (ftd *FakeTiDBControl) GetInfo(tc *v1alpha1.TidbCluster, ordinal int32) (*dbInfo, error) {
return ftd.tidbInfo, ftd.getInfoError
}
Expand Down
19 changes: 0 additions & 19 deletions pkg/manager/member/tidb_upgrader.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,11 +21,6 @@ import (
glog "k8s.io/klog"
)

const (
// MaxResignDDLOwnerCount is the max regign DDL owner count
MaxResignDDLOwnerCount = 3
)

type tidbUpgrader struct {
podLister corelisters.PodLister
tidbControl controller.TiDBControlInterface
Expand Down Expand Up @@ -96,20 +91,6 @@ func (tdu *tidbUpgrader) Upgrade(tc *v1alpha1.TidbCluster, oldSet *apps.Stateful
}

func (tdu *tidbUpgrader) upgradeTiDBPod(tc *v1alpha1.TidbCluster, ordinal int32, newSet *apps.StatefulSet) error {
tcName := tc.GetName()
if tc.Spec.TiDB.Replicas > 1 {
if member, exist := tc.Status.TiDB.Members[tidbPodName(tcName, ordinal)]; exist && member.Health {
hasResign, err := tdu.tidbControl.ResignDDLOwner(tc, ordinal)
if (!hasResign || err != nil) && tc.Status.TiDB.ResignDDLOwnerRetryCount < MaxResignDDLOwnerCount {
glog.Errorf("tidb upgrader: failed to resign ddl owner to %s, %v", member.Name, err)
tc.Status.TiDB.ResignDDLOwnerRetryCount++
return err
}
glog.Infof("tidb upgrader: resign ddl owner to %s successfully", member.Name)
}
}

tc.Status.TiDB.ResignDDLOwnerRetryCount = 0
setUpgradePartition(newSet, ordinal)
return nil
}
Expand Down
41 changes: 1 addition & 40 deletions pkg/manager/member/tidb_upgrader_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,6 @@
package member

import (
"fmt"
"testing"

. "github.com/onsi/gomega"
Expand All @@ -37,21 +36,14 @@ func TestTiDBUpgrader_Upgrade(t *testing.T) {
name string
changeFn func(*v1alpha1.TidbCluster)
getLastAppliedConfigErr bool
resignDDLOwnerError bool
errorExpect bool
changeOldSet func(set *apps.StatefulSet)
expectFn func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet)
}

testFn := func(test *testcase, t *testing.T) {
t.Log(test.name)
upgrader, tidbControl, podInformer := newTiDBUpgrader()
if test.resignDDLOwnerError {
tidbControl.SetResignDDLOwnerError(fmt.Errorf("resign DDL owner failed"))
tidbControl.NotDDLOwner(false)
} else {
tidbControl.NotDDLOwner(true)
}
upgrader, _, podInformer := newTiDBUpgrader()
tc := newTidbClusterForTiDBUpgrader()
if test.changeFn != nil {
test.changeFn(tc)
Expand Down Expand Up @@ -192,37 +184,6 @@ func TestTiDBUpgrader_Upgrade(t *testing.T) {
g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal(controller.Int32Ptr(1)))
},
},
{
name: "resign DDL owner error",
changeFn: func(tc *v1alpha1.TidbCluster) {
tc.Status.PD.Phase = v1alpha1.NormalPhase
tc.Status.TiKV.Phase = v1alpha1.NormalPhase
},
getLastAppliedConfigErr: false,
resignDDLOwnerError: true,
errorExpect: true,
expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) {
g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.UpgradePhase))
g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal(controller.Int32Ptr(1)))
g.Expect(tc.Status.TiDB.ResignDDLOwnerRetryCount).To(Equal(int32(1)))
},
},
{
name: "resign DDL owner error count larger than MaxResignDDLOwnerCount",
changeFn: func(tc *v1alpha1.TidbCluster) {
tc.Status.PD.Phase = v1alpha1.NormalPhase
tc.Status.TiKV.Phase = v1alpha1.NormalPhase
tc.Status.TiDB.ResignDDLOwnerRetryCount = 4
},
getLastAppliedConfigErr: false,
resignDDLOwnerError: true,
errorExpect: false,
expectFn: func(g *GomegaWithT, tc *v1alpha1.TidbCluster, newSet *apps.StatefulSet) {
g.Expect(tc.Status.TiDB.Phase).To(Equal(v1alpha1.UpgradePhase))
g.Expect(newSet.Spec.UpdateStrategy.RollingUpdate.Partition).To(Equal(controller.Int32Ptr(0)))
g.Expect(tc.Status.TiDB.ResignDDLOwnerRetryCount).To(Equal(int32(0)))
},
},
}

for _, test := range tests {
Expand Down
32 changes: 1 addition & 31 deletions tests/pkg/webhook/pods.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,12 @@ package webhook
import (
"fmt"
"os"
"strconv"
"strings"
"time"

"github.com/pingcap/tidb-operator/tests/slack"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/pingcap/tidb-operator/pkg/controller"
"github.com/pingcap/tidb-operator/pkg/label"
"github.com/pingcap/tidb-operator/pkg/pdapi"
"github.com/pingcap/tidb-operator/tests/pkg/client"
Expand Down Expand Up @@ -62,7 +59,6 @@ func (wh *webhook) admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionRespo
}

pdClient := pdapi.NewDefaultPDControl(kubeCli).GetPDClient(pdapi.Namespace(tc.GetNamespace()), tc.GetName(), tc.Spec.EnableTLSCluster)
tidbController := controller.NewDefaultTiDBControl()

// if pod is already deleting, return Allowed
if pod.DeletionTimestamp != nil {
Expand All @@ -71,33 +67,7 @@ func (wh *webhook) admitPods(ar v1beta1.AdmissionReview) *v1beta1.AdmissionRespo
return &reviewResponse
}

if pod.Labels[label.ComponentLabelKey] == "tidb" {
ordinal, err := strconv.ParseInt(strings.Split(name, "-")[len(strings.Split(name, "-"))-1], 10, 32)
if err != nil {
glog.Errorf("fail to convert string to int while deleting TiDB err %v", err)
return &reviewResponse
}

info, err := tidbController.GetInfo(tc, int32(ordinal))
if err != nil {
glog.Errorf("fail to get tidb info error:%v", err)
return &reviewResponse
}

if info.IsOwner && tc.Status.TiDB.StatefulSet.Replicas > 1 {
time.Sleep(10 * time.Second)
err := fmt.Errorf("tidb is ddl owner, can't be deleted namespace %s name %s", namespace, name)
glog.Error(err)
sendErr := slack.SendErrMsg(err.Error())
if sendErr != nil {
glog.Error(sendErr)
}
// TODO use context instead
os.Exit(3)
}
glog.Infof("savely delete pod namespace %s name %s isowner %t", namespace, name, info.IsOwner)

} else if pod.Labels[label.ComponentLabelKey] == "pd" {
if pod.Labels[label.ComponentLabelKey] == "pd" {

leader, err := pdClient.GetPDLeader()
if err != nil {
Expand Down

0 comments on commit 5e306b2

Please sign in to comment.