mvcc/backend: Delete orphaned db.tmp files before defrag

mvcc/backend/backend.go

@@ -358,7 +358,16 @@ func (b *backend) defrag() error {
 
 	b.batchTx.tx = nil
 
-	tmpdb, err := bolt.Open(b.db.Path()+".tmp", 0600, boltOpenOptions)
+	tmpdbPath := b.db.Path() + ".tmp"
+	if _, err := os.Stat(tmpdbPath); err == nil {
+		// Ensure we start with a clean slate since
+		// db.tmp files can be orphaned if etcd is terminated
+		// before defrag completes
+		if rmErr := os.Remove(tmpdbPath); rmErr != nil && !os.IsNotExist(rmErr) {
+			return fmt.Errorf("failed to remove db.tmp before starting defragmentation: %v", err)
+		}
+	}
+	tmpdb, err := bolt.Open(tmpdbPath, 0600, boltOpenOptions)
 	if err != nil {
 		return err
 	}
@@ -376,12 +385,12 @@ func (b *backend) defrag() error {
 			zap.String("current-db-size-in-use", humanize.Bytes(uint64(sizeInUse1))),
 		)
 	}
-
+	// gofail: var defragBeforeCopy struct{}
 	err = defragdb(b.db, tmpdb, defragLimit)
 	if err != nil {
 		tmpdb.Close()
 		if rmErr := os.RemoveAll(tmpdb.Path()); rmErr != nil {
-			b.lg.Error("failed to remove dirs under tmpdb", zap.Error(rmErr))
+			b.lg.Error("failed to remove db.tmp after defragmentation completed", zap.Error(rmErr))
 		}
 		return err
 	}
@@ -394,6 +403,7 @@ func (b *backend) defrag() error {
 	if err != nil {
 		b.lg.Fatal("failed to close tmp database", zap.Error(err))
 	}
+	// gofail: var defragBeforeRename struct{}
 	err = os.Rename(tdbp, dbp)
 	if err != nil {
 		b.lg.Fatal("failed to rename tmp database", zap.Error(err))