Merge branch 'batch-restore' of ssh://github.com/pingcap/br into batc…

…h-restore Signed-off-by: Neil Shen <overvenus@gmail.com>
pingcap · Feb 24, 2020 · a8a13b7 · a8a13b7
2 parents 704c004 + 8810be1
commit a8a13b7
Show file tree

Hide file tree

Showing 16 changed files with 163 additions and 215 deletions.
diff --git a/cmd/validate.go b/cmd/validate.go
@@ -19,9 +19,9 @@ import (
 	"go.uber.org/zap"
 
 	"github.com/pingcap/br/pkg/restore"
+	"github.com/pingcap/br/pkg/rtree"
 	"github.com/pingcap/br/pkg/task"
 	"github.com/pingcap/br/pkg/utils"
-	"github.com/pingcap/br/pkg/utils/rtree"
 )
 
 // NewValidateCommand return a debug subcommand.

diff --git a/pkg/backup/client.go b/pkg/backup/client.go
@@ -29,10 +29,10 @@ import (
 	"github.com/pingcap/tidb/util/ranger"
 	"go.uber.org/zap"
 
+	"github.com/pingcap/br/pkg/rtree"
 	"github.com/pingcap/br/pkg/storage"
 	"github.com/pingcap/br/pkg/summary"
 	"github.com/pingcap/br/pkg/utils"
-	"github.com/pingcap/br/pkg/utils/rtree"
 )
 
 // ClientMgr manages connections needed by backup.
@@ -430,7 +430,7 @@ func (bc *Client) backupRange(
 	})
 
 	// Check if there are duplicated files.
-	checkDupFiles(&results)
+	rtree.CheckDupFiles(&results)
 
 	return nil
 }
@@ -474,7 +474,7 @@ func (bc *Client) fineGrainedBackup(
 	bo := tikv.NewBackoffer(ctx, backupFineGrainedMaxBackoff)
 	for {
 		// Step1, check whether there is any incomplete range
-		incomplete := getIncompleteRange(&rangeTree, startKey, endKey)
+		incomplete := rangeTree.GetIncompleteRange(startKey, endKey)
 		if len(incomplete) == 0 {
 			return nil
 		}

diff --git a/pkg/backup/push.go b/pkg/backup/push.go
@@ -10,7 +10,7 @@ import (
 	"github.com/pingcap/log"
 	"go.uber.org/zap"
 
-	"github.com/pingcap/br/pkg/utils/rtree"
+	"github.com/pingcap/br/pkg/rtree"
 )
 
 // pushDown warps a backup task.

diff --git a/pkg/backup/range.go b/pkg/backup/range.go
diff --git a/pkg/restore/range.go b/pkg/restore/range.go
@@ -8,7 +8,7 @@ import (
 	"github.com/pingcap/tidb/tablecodec"
 	"go.uber.org/zap"
 
-	"github.com/pingcap/br/pkg/utils/rtree"
+	"github.com/pingcap/br/pkg/rtree"
 )
 
 // sortRanges checks if the range overlapped and sort them

diff --git a/pkg/restore/range_test.go b/pkg/restore/range_test.go
@@ -7,7 +7,7 @@ import (
 	"github.com/pingcap/kvproto/pkg/import_sstpb"
 	"github.com/pingcap/tidb/tablecodec"
 
-	"github.com/pingcap/br/pkg/utils/rtree"
+	"github.com/pingcap/br/pkg/rtree"
 )
 
 type testRangeSuite struct{}

diff --git a/pkg/restore/split.go b/pkg/restore/split.go
@@ -14,7 +14,7 @@ import (
 	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
 
-	"github.com/pingcap/br/pkg/utils/rtree"
+	"github.com/pingcap/br/pkg/rtree"
 )
 
 // Constants for split retry machinery.

diff --git a/pkg/restore/split_test.go b/pkg/restore/split_test.go
@@ -14,7 +14,7 @@ import (
 	"github.com/pingcap/pd/server/schedule/placement"
 	"github.com/pingcap/tidb/util/codec"
 
-	"github.com/pingcap/br/pkg/utils/rtree"
+	"github.com/pingcap/br/pkg/rtree"
 )
 
 type testClient struct {

diff --git a/pkg/restore/util.go b/pkg/restore/util.go
@@ -18,8 +18,8 @@ import (
 	"github.com/pingcap/tidb/util/codec"
 	"go.uber.org/zap"
 
+	"github.com/pingcap/br/pkg/rtree"
 	"github.com/pingcap/br/pkg/summary"
-	"github.com/pingcap/br/pkg/utils/rtree"
 )
 
 var recordPrefixSep = []byte("_r")

diff --git a/pkg/rtree/check.go b/pkg/rtree/check.go
@@ -0,0 +1,31 @@
+package rtree
+
+import (
+	"encoding/hex"
+
+	"github.com/google/btree"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
+)
+
+// CheckDupFiles checks if there are any files are duplicated.
+func CheckDupFiles(rangeTree *RangeTree) {
+	// Name -> SHA256
+	files := make(map[string][]byte)
+	rangeTree.Ascend(func(i btree.Item) bool {
+		rg := i.(*Range)
+		for _, f := range rg.Files {
+			old, ok := files[f.Name]
+			if ok {
+				log.Error("dup file",
+					zap.String("Name", f.Name),
+					zap.String("SHA256_1", hex.EncodeToString(old)),
+					zap.String("SHA256_2", hex.EncodeToString(f.Sha256)),
+				)
+			} else {
+				files[f.Name] = f.Sha256
+			}
+		}
+		return true
+	})
+}
diff --git a/pkg/utils/rtree/rtree.go → pkg/rtree/rtree.go b/pkg/utils/rtree/rtree.go → pkg/rtree/rtree.go
@@ -28,8 +28,6 @@ type Range struct {
 	StartKey []byte
 	EndKey   []byte
 	Files    []*backup.File
-	// Addition data attached in Range
-	// Attachment interface{}
 }
 
 // String formats a range to a string
@@ -187,3 +185,44 @@ func (rangeTree *RangeTree) GetSortedRanges() []Range {
 	})
 	return sortedRanges
 }
+
+// GetIncompleteRange returns missing range covered by startKey and endKey.
+func (rangeTree *RangeTree) GetIncompleteRange(
+	startKey, endKey []byte,
+) []Range {
+	if len(startKey) != 0 && bytes.Equal(startKey, endKey) {
+		return []Range{}
+	}
+	incomplete := make([]Range, 0, 64)
+	requsetRange := Range{StartKey: startKey, EndKey: endKey}
+	lastEndKey := startKey
+	pviot := &Range{StartKey: startKey}
+	if first := rangeTree.Find(pviot); first != nil {
+		pviot.StartKey = first.StartKey
+	}
+	rangeTree.AscendGreaterOrEqual(pviot, func(i btree.Item) bool {
+		rg := i.(*Range)
+		if bytes.Compare(lastEndKey, rg.StartKey) < 0 {
+			start, end, isIntersect :=
+				requsetRange.Intersect(lastEndKey, rg.StartKey)
+			if isIntersect {
+				// There is a gap between the last item and the current item.
+				incomplete =
+					append(incomplete, Range{StartKey: start, EndKey: end})
+			}
+		}
+		lastEndKey = rg.EndKey
+		return len(endKey) == 0 || bytes.Compare(rg.EndKey, endKey) < 0
+	})
+
+	// Check whether we need append the last range
+	if !bytes.Equal(lastEndKey, endKey) && len(lastEndKey) != 0 &&
+		(len(endKey) == 0 || bytes.Compare(lastEndKey, endKey) < 0) {
+		start, end, isIntersect := requsetRange.Intersect(lastEndKey, endKey)
+		if isIntersect {
+			incomplete =
+				append(incomplete, Range{StartKey: start, EndKey: end})
+		}
+	}
+	return incomplete
+}