Merge branch 'master' into master

.github/workflows/install.yaml

@@ -41,10 +41,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/integrate-cluster-cmd.yaml

@@ -43,10 +43,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/integrate-cluster-scale.yaml

@@ -42,10 +42,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/integrate-dm.yaml

@@ -42,10 +42,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/integrate-playground.yaml

@@ -41,10 +41,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/integrate-tiup.yaml

@@ -45,10 +45,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory
@@ -90,10 +90,10 @@ jobs:
           ref: ${{ github.event.pull_request.head.sha }}
           path: go/src/github.com/${{ github.repository }}
 
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: make unit-test

.github/workflows/release-tiup.yaml

@@ -41,10 +41,10 @@ jobs:
     env:
       working-directory: ${{ github.workspace }}/go/src/github.com/${{ github.repository }}
     steps:
-      - name: Set up Go 1.19
-        uses: actions/setup-go@v2
+      - name: Set up Go 1.21
+        uses: actions/setup-go@v4
         with:
-          go-version: 1.19.x
+          go-version: 1.21.x
         id: go
 
       - name: Check out code into the Go module directory

.github/workflows/reprotest.yaml

@@ -35,14 +35,14 @@ jobs:
     strategy:
       matrix:
         go:
-          - 1.19.x
+          - 1.21.x
 
     # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
       - uses: actions/checkout@v3
 
       - name: Set up Go ${{ matrix.go }}
-        uses: actions/setup-go@v2
+        uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go }}
 

Makefile

@@ -150,5 +150,5 @@ tools/bin/revive: tools/check/go.mod
 	$(GO) build -o ../bin/revive github.com/mgechev/revive
 
 tools/bin/golangci-lint:
-	@# Target: pull in specific version of golangci-lint (v1.50.0)
-	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./tools/bin v1.50.0
+	@# Target: pull in specific version of golangci-lint (v1.54.1)
+	curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./tools/bin v1.54.1

components/client/go.mod

@@ -1,6 +1,6 @@
 module github.com/tiup/components/client
 
-go 1.18
+go 1.21
 
 require (
 	github.com/gizak/termui/v3 v3.1.0

components/dm/spec/logic.go

@@ -460,7 +460,7 @@ func (topo *Specification) ComponentsByStartOrder() (comps []Component) {
 }
 
 // ComponentsByUpdateOrder return component in the order need to be updated.
-func (topo *Specification) ComponentsByUpdateOrder() (comps []Component) {
+func (topo *Specification) ComponentsByUpdateOrder(curVer string) (comps []Component) {
 	// "dm-master", "dm-worker"
 	comps = append(comps, &DMMasterComponent{topo})
 	comps = append(comps, &DMWorkerComponent{topo})

doc/dev/README.md

@@ -6,7 +6,7 @@ You can build TiUP on any platform that supports Go.
 
 Prerequisites:
 
-* Go (minimum version: 1.18; [installation instructions](https://golang.org/doc/install))
+* Go (minimum version: 1.21; [installation instructions](https://golang.org/doc/install))
 * golint (`go get -u golang.org/x/lint/golint`)
 * make
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/pingcap/tiup
 
-go 1.18
+go 1.21
 
 replace gopkg.in/yaml.v2 => github.com/july2993/yaml v0.0.0-20200423062752-adcfa5abe2ed
 

go.sum

@@ -85,6 +85,7 @@ github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=

pkg/cluster/manager/check.go

@@ -190,7 +190,7 @@ func checkSystemInfo(
 
 	roleFilter := set.NewStringSet(gOpt.Roles...)
 	nodeFilter := set.NewStringSet(gOpt.Nodes...)
-	components := topo.ComponentsByUpdateOrder()
+	components := topo.ComponentsByStartOrder()
 	components = operator.FilterComponent(components, roleFilter)
 
 	for _, comp := range components {

pkg/cluster/manager/patch.go

@@ -19,6 +19,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strings"
 
 	"github.com/fatih/color"
@@ -204,7 +205,13 @@ func overwritePatch(specManager *spec.SpecManager, name, comp, packagePath strin
 	if utils.IsSymExist(symlink) {
 		os.Remove(symlink)
 	}
-	return os.Symlink(tg, symlink)
+
+	tgRelPath, err := filepath.Rel(filepath.Dir(symlink), tg)
+	if err != nil {
+		return err
+	}
+
+	return os.Symlink(tgRelPath, symlink)
 }
 
 func instancesToPatch(topo spec.Topology, options operator.Options) ([]spec.Instance, error) {

pkg/cluster/manager/upgrade.go

@@ -87,7 +87,7 @@ Do you want to continue? [y/N]:`,
 	}
 
 	hasImported := false
-	for _, comp := range topo.ComponentsByUpdateOrder() {
+	for _, comp := range topo.ComponentsByUpdateOrder(base.Version) {
 		for _, inst := range comp.Instances() {
 			compName := inst.ComponentName()
 

pkg/cluster/operation/destroy.go

@@ -23,6 +23,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/fatih/color"
 	perrs "github.com/pingcap/errors"
 	"github.com/pingcap/tiup/pkg/cluster/api"
 	"github.com/pingcap/tiup/pkg/cluster/ctxt"
@@ -434,27 +435,23 @@ func DestroyComponent(ctx context.Context, instances []spec.Instance, cls spec.T
 			delPaths.Insert(fmt.Sprintf("/etc/systemd/system/%s", svc))
 		}
 		logger.Debugf("Deleting paths on %s: %s", ins.GetManageHost(), strings.Join(delPaths.Slice(), " "))
-		c := module.ShellModuleConfig{
-			Command:  fmt.Sprintf("rm -rf %s;", strings.Join(delPaths.Slice(), " ")),
-			Sudo:     true, // the .service files are in a directory owned by root
-			Chdir:    "",
-			UseShell: false,
-		}
-		shell := module.NewShellModule(c)
-		stdout, stderr, err := shell.Execute(ctx, e)
-
-		if len(stdout) > 0 {
-			fmt.Println(string(stdout))
-		}
-		if len(stderr) > 0 {
-			logger.Errorf(string(stderr))
-		}
+		for _, delPath := range delPaths.Slice() {
+			c := module.ShellModuleConfig{
+				Command:  fmt.Sprintf("rm -rf %s;", delPath),
+				Sudo:     true, // the .service files are in a directory owned by root
+				Chdir:    "",
+				UseShell: false,
+			}
+			shell := module.NewShellModule(c)
+			_, _, err := shell.Execute(ctx, e)
 
-		if err != nil {
-			return perrs.Annotatef(err, "failed to destroy: %s", ins.GetManageHost())
+			if err != nil {
+				// Ignore error and continue.For example, deleting a mount point will result in a "Device or resource busy" error.
+				logger.Warnf(color.YellowString("Warn: failed to delete path \"%s\" on %s.Please check this error message and manually delete if necessary.\nerrmsg: %s", delPath, ins.GetManageHost(), err))
+			}
 		}
 
-		logger.Infof("Destroy %s success", ins.GetManageHost())
+		logger.Infof("Destroy %s finished", ins.GetManageHost())
 		logger.Infof("- Destroy %s paths: %v", ins.ComponentName(), delPaths.Slice())
 	}
 

pkg/cluster/operation/upgrade.go

@@ -49,7 +49,7 @@ func Upgrade(
 ) error {
 	roleFilter := set.NewStringSet(options.Roles...)
 	nodeFilter := set.NewStringSet(options.Nodes...)
-	components := topo.ComponentsByUpdateOrder()
+	components := topo.ComponentsByUpdateOrder(currentVersion)
 	components = FilterComponent(components, roleFilter)
 	logger := ctx.Value(logprinter.ContextKeyLogger).(*logprinter.Logger)
 

pkg/cluster/spec/grafana.go

@@ -319,7 +319,7 @@ func (i *GrafanaInstance) installDashboards(ctx context.Context, e ctxt.Executor
 		return errors.Annotatef(err, "stderr: %s", string(stderr))
 	}
 
-	srcPath := PackagePath(ComponentDMMaster, clusterVersion, i.OS(), i.Arch())
+	srcPath := PackagePath(GetDMMasterPackageName(i.topo), clusterVersion, i.OS(), i.Arch())
 	dstPath := filepath.Join(tmp, filepath.Base(srcPath))
 	err = e.Transfer(ctx, srcPath, dstPath, false, 0, false)
 	if err != nil {

pkg/cluster/spec/monitoring.go

@@ -439,7 +439,7 @@ func (i *MonitorInstance) installRules(ctx context.Context, e ctxt.Executor, dep
 		return errors.Annotatef(err, "stderr: %s", string(stderr))
 	}
 
-	srcPath := PackagePath(ComponentDMMaster, clusterVersion, i.OS(), i.Arch())
+	srcPath := PackagePath(GetDMMasterPackageName(i.topo), clusterVersion, i.OS(), i.Arch())
 	dstPath := filepath.Join(tmp, filepath.Base(srcPath))
 
 	err = e.Transfer(ctx, srcPath, dstPath, false, 0, false)

pkg/cluster/spec/spec.go

@@ -161,7 +161,7 @@ type Topology interface {
 	// Instances() []Instance
 	ComponentsByStartOrder() []Component
 	ComponentsByStopOrder() []Component
-	ComponentsByUpdateOrder() []Component
+	ComponentsByUpdateOrder(curVer string) []Component
 	IterInstance(fn func(instance Instance), concurrency ...int)
 	GetMonitoredOptions() *MonitoredOptions
 	// count how many time a path is used by instances in cluster
@@ -728,16 +728,24 @@ func (s *Specification) ComponentsByStartOrder() (comps []Component) {
 }
 
 // ComponentsByUpdateOrder return component in the order need to be updated.
-func (s *Specification) ComponentsByUpdateOrder() (comps []Component) {
-	// "tiflash", "pd", "dashboard", "tikv", "pump", "tidb", "drainer", "cdc", "prometheus", "grafana", "alertmanager"
+func (s *Specification) ComponentsByUpdateOrder(curVer string) (comps []Component) {
+	// Ref: https://github.com/pingcap/tiup/issues/2166
+	cdcUpgradeBeforePDTiKVTiDB := tidbver.TiCDCUpgradeBeforePDTiKVTiDB(curVer)
+
+	// "tiflash", <"cdc">, "pd", "dashboard", "tikv", "pump", "tidb", "drainer", <"cdc>", "prometheus", "grafana", "alertmanager"
 	comps = append(comps, &TiFlashComponent{s})
+	if cdcUpgradeBeforePDTiKVTiDB {
+		comps = append(comps, &CDCComponent{s})
+	}
 	comps = append(comps, &PDComponent{s})
 	comps = append(comps, &DashboardComponent{s})
 	comps = append(comps, &TiKVComponent{s})
 	comps = append(comps, &PumpComponent{s})
 	comps = append(comps, &TiDBComponent{s})
 	comps = append(comps, &DrainerComponent{s})
-	comps = append(comps, &CDCComponent{s})
+	if !cdcUpgradeBeforePDTiKVTiDB {
+		comps = append(comps, &CDCComponent{s})
+	}
 	comps = append(comps, &MonitorComponent{s})
 	comps = append(comps, &GrafanaComponent{s})
 	comps = append(comps, &AlertManagerComponent{s})

pkg/cluster/spec/spec_manager_test.go

@@ -97,7 +97,7 @@ func (t *TestTopology) ComponentsByStopOrder() []Component {
 	return nil
 }
 
-func (t *TestTopology) ComponentsByUpdateOrder() []Component {
+func (t *TestTopology) ComponentsByUpdateOrder(curVer string) []Component {
 	return nil
 }
 

pkg/cluster/spec/util.go

@@ -218,3 +218,16 @@ func PackagePath(comp string, version string, os string, arch string) string {
 	fileName := fmt.Sprintf("%s-%s-%s-%s.tar.gz", comp, version, os, arch)
 	return ProfilePath(TiUPPackageCacheDir, fileName)
 }
+
+// GetDMMasterPackageName return package name of the first DMMaster instance
+func GetDMMasterPackageName(topo Topology) string {
+	for _, c := range topo.ComponentsByStartOrder() {
+		if c.Name() == ComponentDMMaster {
+			instances := c.Instances()
+			if len(instances) > 0 {
+				return instances[0].ComponentSource()
+			}
+		}
+	}
+	return ComponentDMMaster
+}

pkg/tidbver/tidbver.go

@@ -130,6 +130,11 @@ func TiCDCSupportRollingUpgrade(version string) bool {
 	return semver.Compare(version, "v6.3.0") >= 0 || strings.Contains(version, "nightly")
 }
 
+// TiCDCUpgradeBeforePDTiKVTiDB return if the given version of TiCDC should upgrade TiCDC before PD and TiKV
+func TiCDCUpgradeBeforePDTiKVTiDB(version string) bool {
+	return semver.Compare(version, "v5.1.0") >= 0 || strings.Contains(version, "nightly")
+}
+
 // NgMonitorDeployByDefault return if given version of TiDB cluster should contain ng-monitoring
 func NgMonitorDeployByDefault(version string) bool {
 	return semver.Compare(version, "v5.4.0") >= 0 || strings.Contains(version, "nightly")