Splic pkg/restic package

This commit splits the pkg/restic package into several packages to support Kopia integration works

Fixes #5055

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>

Tiltfile

@@ -7,7 +7,7 @@ k8s_yaml([
     'config/crd/v1/bases/velero.io_downloadrequests.yaml',
     'config/crd/v1/bases/velero.io_podvolumebackups.yaml',
     'config/crd/v1/bases/velero.io_podvolumerestores.yaml',
-    'config/crd/v1/bases/velero.io_resticrepositories.yaml',
+    'config/crd/v1/bases/velero.io_backuprepositories.yaml',
     'config/crd/v1/bases/velero.io_restores.yaml',
     'config/crd/v1/bases/velero.io_schedules.yaml',
     'config/crd/v1/bases/velero.io_serverstatusrequests.yaml',

pkg/backup/backup.go

@@ -46,7 +46,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/podexec"
-	"github.com/vmware-tanzu/velero/pkg/restic"
+	"github.com/vmware-tanzu/velero/pkg/uploader"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 )
@@ -74,7 +74,7 @@ type kubernetesBackupper struct {
 	dynamicFactory         client.DynamicFactory
 	discoveryHelper        discovery.Helper
 	podCommandExecutor     podexec.PodCommandExecutor
-	resticBackupperFactory restic.BackupperFactory
+	resticBackupperFactory uploader.BackupperFactory
 	resticTimeout          time.Duration
 	defaultVolumesToRestic bool
 	clientPageSize         int
@@ -100,7 +100,7 @@ func NewKubernetesBackupper(
 	discoveryHelper discovery.Helper,
 	dynamicFactory client.DynamicFactory,
 	podCommandExecutor podexec.PodCommandExecutor,
-	resticBackupperFactory restic.BackupperFactory,
+	resticBackupperFactory uploader.BackupperFactory,
 	resticTimeout time.Duration,
 	defaultVolumesToRestic bool,
 	clientPageSize int,
@@ -234,7 +234,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 	ctx, cancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout)
 	defer cancelFunc()
 
-	var resticBackupper restic.Backupper
+	var resticBackupper uploader.Backupper
 	if kb.resticBackupperFactory != nil {
 		resticBackupper, err = kb.resticBackupperFactory.NewBackupper(ctx, backupRequest.Backup)
 		if err != nil {

pkg/backup/backup_test.go

@@ -47,9 +47,9 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/discovery"
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
-	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/test"
 	testutil "github.com/vmware-tanzu/velero/pkg/test"
+	"github.com/vmware-tanzu/velero/pkg/uploader"
 	kubeutil "github.com/vmware-tanzu/velero/pkg/util/kube"
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
@@ -2595,7 +2595,7 @@ func TestBackupWithHooks(t *testing.T) {
 
 type fakeResticBackupperFactory struct{}
 
-func (f *fakeResticBackupperFactory) NewBackupper(context.Context, *velerov1.Backup) (restic.Backupper, error) {
+func (f *fakeResticBackupperFactory) NewBackupper(context.Context, *velerov1.Backup) (uploader.Backupper, error) {
 	return &fakeResticBackupper{}, nil
 }
 

pkg/backup/item_backupper.go

@@ -43,6 +43,7 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/kuberesource"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/restic"
+	"github.com/vmware-tanzu/velero/pkg/uploader"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/volume"
 )
@@ -53,7 +54,7 @@ type itemBackupper struct {
 	tarWriter               tarWriter
 	dynamicFactory          client.DynamicFactory
 	discoveryHelper         discovery.Helper
-	resticBackupper         restic.Backupper
+	resticBackupper         uploader.Backupper
 	resticSnapshotTracker   *pvcSnapshotTracker
 	volumeSnapshotterGetter VolumeSnapshotterGetter
 

pkg/controller/restic_repository_controller.go

@@ -26,7 +26,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/clock"
-
 	ctrl "sigs.k8s.io/controller-runtime"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 

pkg/repository/config/config.go

@@ -34,9 +34,7 @@ const (
 	AzureBackend BackendType = "velero.io/azure"
 	GCPBackend   BackendType = "velero.io/gcp"
 	FSBackend    BackendType = "velero.io/fs"
-)
 
-const (
 	// CredentialsFileKey is the key within a BSL config that is checked to see if
 	// the BSL is using its own credentials, rather than those in the environment
 	CredentialsFileKey = "credentialsFile"

pkg/restic/repository_ensurer.go → pkg/repository/ensurer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repository
 
 import (
 	"context"
@@ -35,8 +35,8 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/label"
 )
 
-// repositoryEnsurer ensures that Velero restic repositories are created and ready.
-type repositoryEnsurer struct {
+// RepositoryEnsurer ensures that backup repositories are created and ready.
+type RepositoryEnsurer struct {
 	log        logrus.FieldLogger
 	repoLister velerov1listers.BackupRepositoryLister
 	repoClient velerov1client.BackupRepositoriesGetter
@@ -55,8 +55,8 @@ type repoKey struct {
 	backupLocation  string
 }
 
-func newRepositoryEnsurer(repoInformer velerov1informers.BackupRepositoryInformer, repoClient velerov1client.BackupRepositoriesGetter, log logrus.FieldLogger) *repositoryEnsurer {
-	r := &repositoryEnsurer{
+func NewRepositoryEnsurer(repoInformer velerov1informers.BackupRepositoryInformer, repoClient velerov1client.BackupRepositoriesGetter, log logrus.FieldLogger) *RepositoryEnsurer {
+	r := &RepositoryEnsurer{
 		log:        log,
 		repoLister: repoInformer.Lister(),
 		repoClient: repoClient,
@@ -105,7 +105,7 @@ func repoLabels(volumeNamespace, backupLocation string) labels.Set {
 	}
 }
 
-func (r *repositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, backupLocation string) (*velerov1api.BackupRepository, error) {
+func (r *RepositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, backupLocation string) (*velerov1api.BackupRepository, error) {
 	log := r.log.WithField("volumeNamespace", volumeNamespace).WithField("backupLocation", backupLocation)
 
 	// It's only safe to have one instance of this method executing concurrently for a
@@ -190,15 +190,15 @@ func (r *repositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNam
 	}
 }
 
-func (r *repositoryEnsurer) getRepoChan(name string) chan *velerov1api.BackupRepository {
+func (r *RepositoryEnsurer) getRepoChan(name string) chan *velerov1api.BackupRepository {
 	r.repoChansLock.Lock()
 	defer r.repoChansLock.Unlock()
 
 	r.repoChans[name] = make(chan *velerov1api.BackupRepository)
 	return r.repoChans[name]
 }
 
-func (r *repositoryEnsurer) repoLock(volumeNamespace, backupLocation string) *sync.Mutex {
+func (r *RepositoryEnsurer) repoLock(volumeNamespace, backupLocation string) *sync.Mutex {
 	r.repoLocksMu.Lock()
 	defer r.repoLocksMu.Unlock()
 

pkg/restic/repo_locker.go → pkg/repository/locker.go

@@ -13,52 +13,53 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package restic
+
+package repository
 
 import "sync"
 
-// repoLocker manages exclusive/non-exclusive locks for
+// RepoLocker manages exclusive/non-exclusive locks for
 // operations against restic repositories. The semantics
 // of exclusive/non-exclusive locks are the same as for
 // a sync.RWMutex, where a non-exclusive lock is equivalent
 // to a read lock, and an exclusive lock is equivalent to
 // a write lock.
-type repoLocker struct {
+type RepoLocker struct {
 	mu    sync.Mutex
 	locks map[string]*sync.RWMutex
 }
 
-func newRepoLocker() *repoLocker {
-	return &repoLocker{
+func NewRepoLocker() *RepoLocker {
+	return &RepoLocker{
 		locks: make(map[string]*sync.RWMutex),
 	}
 }
 
 // LockExclusive acquires an exclusive lock for the specified
 // repository. This function blocks until no other locks exist
 // for the repo.
-func (rl *repoLocker) LockExclusive(name string) {
+func (rl *RepoLocker) LockExclusive(name string) {
 	rl.ensureLock(name).Lock()
 }
 
 // Lock acquires a non-exclusive lock for the specified
 // repository. This function blocks until no exclusive
 // locks exist for the repo.
-func (rl *repoLocker) Lock(name string) {
+func (rl *RepoLocker) Lock(name string) {
 	rl.ensureLock(name).RLock()
 }
 
 // UnlockExclusive releases an exclusive lock for the repo.
-func (rl *repoLocker) UnlockExclusive(name string) {
+func (rl *RepoLocker) UnlockExclusive(name string) {
 	rl.ensureLock(name).Unlock()
 }
 
 // Unlock releases a non-exclusive lock for the repo.
-func (rl *repoLocker) Unlock(name string) {
+func (rl *RepoLocker) Unlock(name string) {
 	rl.ensureLock(name).RUnlock()
 }
 
-func (rl *repoLocker) ensureLock(name string) *sync.RWMutex {
+func (rl *RepoLocker) ensureLock(name string) *sync.RWMutex {
 	rl.mu.Lock()
 	defer rl.mu.Unlock()
 

pkg/restic/common.go

@@ -52,99 +52,15 @@ const (
 	// take backup of all pod volumes.
 	DefaultVolumesToRestic = false
 
-	// PVCNameAnnotation is the key for the annotation added to
-	// pod volume backups when they're for a PVC.
-	PVCNameAnnotation = "velero.io/pvc-name"
-
 	// VolumesToBackupAnnotation is the annotation on a pod whose mounted volumes
 	// need to be backed up using restic.
 	VolumesToBackupAnnotation = "backup.velero.io/backup-volumes"
 
 	// VolumesToExcludeAnnotation is the annotation on a pod whose mounted volumes
 	// should be excluded from restic backup.
 	VolumesToExcludeAnnotation = "backup.velero.io/backup-volumes-excludes"
-
-	// Deprecated.
-	//
-	// TODO(2.0): remove
-	podAnnotationPrefix = "snapshot.velero.io/"
 )
 
-// getPodSnapshotAnnotations returns a map, of volume name -> snapshot id,
-// of all restic snapshots for this pod.
-// TODO(2.0) to remove
-// Deprecated: we will stop using pod annotations to record restic snapshot IDs after they're taken,
-// therefore we won't need to check if these annotations exist.
-func getPodSnapshotAnnotations(obj metav1.Object) map[string]string {
-	var res map[string]string
-
-	insertSafe := func(k, v string) {
-		if res == nil {
-			res = make(map[string]string)
-		}
-		res[k] = v
-	}
-
-	for k, v := range obj.GetAnnotations() {
-		if strings.HasPrefix(k, podAnnotationPrefix) {
-			insertSafe(k[len(podAnnotationPrefix):], v)
-		}
-	}
-
-	return res
-}
-
-func isPVBMatchPod(pvb *velerov1api.PodVolumeBackup, podName string, namespace string) bool {
-	return podName == pvb.Spec.Pod.Name && namespace == pvb.Spec.Pod.Namespace
-}
-
-// volumeHasNonRestorableSource checks if the given volume exists in the list of podVolumes
-// and returns true if the volume's source is not restorable. This is true for volumes with
-// a Projected or DownwardAPI source.
-func volumeHasNonRestorableSource(volumeName string, podVolumes []corev1api.Volume) bool {
-	var volume corev1api.Volume
-	for _, v := range podVolumes {
-		if v.Name == volumeName {
-			volume = v
-			break
-		}
-	}
-	return volume.Projected != nil || volume.DownwardAPI != nil
-}
-
-// GetVolumeBackupsForPod returns a map, of volume name -> snapshot id,
-// of the PodVolumeBackups that exist for the provided pod.
-func GetVolumeBackupsForPod(podVolumeBackups []*velerov1api.PodVolumeBackup, pod *corev1api.Pod, sourcePodNs string) map[string]string {
-	volumes := make(map[string]string)
-
-	for _, pvb := range podVolumeBackups {
-		if !isPVBMatchPod(pvb, pod.GetName(), sourcePodNs) {
-			continue
-		}
-
-		// skip PVBs without a snapshot ID since there's nothing
-		// to restore (they could be failed, or for empty volumes).
-		if pvb.Status.SnapshotID == "" {
-			continue
-		}
-
-		// If the volume came from a projected or DownwardAPI source, skip its restore.
-		// This allows backups affected by https://github.com/vmware-tanzu/velero/issues/3863
-		// or https://github.com/vmware-tanzu/velero/issues/4053 to be restored successfully.
-		if volumeHasNonRestorableSource(pvb.Spec.Volume, pod.Spec.Volumes) {
-			continue
-		}
-
-		volumes[pvb.Spec.Volume] = pvb.Status.SnapshotID
-	}
-
-	if len(volumes) > 0 {
-		return volumes
-	}
-
-	return getPodSnapshotAnnotations(pod)
-}
-
 // GetVolumesToBackup returns a list of volume names to backup for
 // the provided pod.
 // Deprecated: Use GetPodVolumesUsingRestic instead.