Splic pkg/restic package

This commit splits the pkg/restic package into several packages to support Kopia integration works

Fixes vmware-tanzu#5055

Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>

pkg/backup/backup.go

@@ -28,6 +28,8 @@ import (
 	"path/filepath"
 	"time"
 
+	"github.com/vmware-tanzu/velero/pkg/uploader"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -46,7 +48,6 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/plugin/framework"
 	"github.com/vmware-tanzu/velero/pkg/plugin/velero"
 	"github.com/vmware-tanzu/velero/pkg/podexec"
-	"github.com/vmware-tanzu/velero/pkg/restic"
 	"github.com/vmware-tanzu/velero/pkg/util/boolptr"
 	"github.com/vmware-tanzu/velero/pkg/util/collections"
 )
@@ -74,7 +75,7 @@ type kubernetesBackupper struct {
 	dynamicFactory         client.DynamicFactory
 	discoveryHelper        discovery.Helper
 	podCommandExecutor     podexec.PodCommandExecutor
-	resticBackupperFactory restic.BackupperFactory
+	resticBackupperFactory uploader.BackupperFactory
 	resticTimeout          time.Duration
 	defaultVolumesToRestic bool
 	clientPageSize         int
@@ -100,7 +101,7 @@ func NewKubernetesBackupper(
 	discoveryHelper discovery.Helper,
 	dynamicFactory client.DynamicFactory,
 	podCommandExecutor podexec.PodCommandExecutor,
-	resticBackupperFactory restic.BackupperFactory,
+	resticBackupperFactory uploader.BackupperFactory,
 	resticTimeout time.Duration,
 	defaultVolumesToRestic bool,
 	clientPageSize int,
@@ -234,7 +235,7 @@ func (kb *kubernetesBackupper) BackupWithResolvers(log logrus.FieldLogger,
 	ctx, cancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout)
 	defer cancelFunc()
 
-	var resticBackupper restic.Backupper
+	var resticBackupper uploader.Backupper
 	if kb.resticBackupperFactory != nil {
 		resticBackupper, err = kb.resticBackupperFactory.NewBackupper(ctx, backupRequest.Backup)
 		if err != nil {

pkg/backup/item_backupper.go

@@ -24,6 +24,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/vmware-tanzu/velero/pkg/uploader"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	corev1api "k8s.io/api/core/v1"
@@ -53,7 +55,7 @@ type itemBackupper struct {
 	tarWriter               tarWriter
 	dynamicFactory          client.DynamicFactory
 	discoveryHelper         discovery.Helper
-	resticBackupper         restic.Backupper
+	resticBackupper         uploader.Backupper
 	resticSnapshotTracker   *pvcSnapshotTracker
 	volumeSnapshotterGetter VolumeSnapshotterGetter
 

pkg/controller/restic_repository_controller.go

@@ -21,6 +21,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/vmware-tanzu/velero/pkg/repository/repoconfig"
+
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -127,7 +129,7 @@ func (r *ResticRepoReconciler) initializeRepo(ctx context.Context, req *velerov1
 		return r.patchResticRepository(ctx, req, repoNotReady(err.Error()))
 	}
 
-	repoIdentifier, err := restic.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
+	repoIdentifier, err := repoconfig.GetRepoIdentifier(loc, req.Spec.VolumeNamespace)
 	if err != nil {
 		return r.patchResticRepository(ctx, req, func(rr *velerov1api.BackupRepository) {
 			rr.Status.Message = err.Error()

pkg/restic/repository_ensurer.go → pkg/repository/ensurer.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repository
 
 import (
 	"context"
@@ -35,8 +35,8 @@ import (
 	"github.com/vmware-tanzu/velero/pkg/label"
 )
 
-// repositoryEnsurer ensures that Velero restic repositories are created and ready.
-type repositoryEnsurer struct {
+// RepositoryEnsurer ensures that backup repositories are created and ready.
+type RepositoryEnsurer struct {
 	log        logrus.FieldLogger
 	repoLister velerov1listers.BackupRepositoryLister
 	repoClient velerov1client.BackupRepositoriesGetter
@@ -55,8 +55,8 @@ type repoKey struct {
 	backupLocation  string
 }
 
-func newRepositoryEnsurer(repoInformer velerov1informers.BackupRepositoryInformer, repoClient velerov1client.BackupRepositoriesGetter, log logrus.FieldLogger) *repositoryEnsurer {
-	r := &repositoryEnsurer{
+func NewRepositoryEnsurer(repoInformer velerov1informers.BackupRepositoryInformer, repoClient velerov1client.BackupRepositoriesGetter, log logrus.FieldLogger) *RepositoryEnsurer {
+	r := &RepositoryEnsurer{
 		log:        log,
 		repoLister: repoInformer.Lister(),
 		repoClient: repoClient,
@@ -105,7 +105,7 @@ func repoLabels(volumeNamespace, backupLocation string) labels.Set {
 	}
 }
 
-func (r *repositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, backupLocation string) (*velerov1api.BackupRepository, error) {
+func (r *RepositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNamespace, backupLocation string) (*velerov1api.BackupRepository, error) {
 	log := r.log.WithField("volumeNamespace", volumeNamespace).WithField("backupLocation", backupLocation)
 
 	// It's only safe to have one instance of this method executing concurrently for a
@@ -190,15 +190,15 @@ func (r *repositoryEnsurer) EnsureRepo(ctx context.Context, namespace, volumeNam
 	}
 }
 
-func (r *repositoryEnsurer) getRepoChan(name string) chan *velerov1api.BackupRepository {
+func (r *RepositoryEnsurer) getRepoChan(name string) chan *velerov1api.BackupRepository {
 	r.repoChansLock.Lock()
 	defer r.repoChansLock.Unlock()
 
 	r.repoChans[name] = make(chan *velerov1api.BackupRepository)
 	return r.repoChans[name]
 }
 
-func (r *repositoryEnsurer) repoLock(volumeNamespace, backupLocation string) *sync.Mutex {
+func (r *RepositoryEnsurer) repoLock(volumeNamespace, backupLocation string) *sync.Mutex {
 	r.repoLocksMu.Lock()
 	defer r.repoLocksMu.Unlock()
 

pkg/restic/repo_locker.go → pkg/repository/locker.go

@@ -13,52 +13,53 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 */
-package restic
+
+package repository
 
 import "sync"
 
-// repoLocker manages exclusive/non-exclusive locks for
+// RepoLocker manages exclusive/non-exclusive locks for
 // operations against restic repositories. The semantics
 // of exclusive/non-exclusive locks are the same as for
 // a sync.RWMutex, where a non-exclusive lock is equivalent
 // to a read lock, and an exclusive lock is equivalent to
 // a write lock.
-type repoLocker struct {
+type RepoLocker struct {
 	mu    sync.Mutex
 	locks map[string]*sync.RWMutex
 }
 
-func newRepoLocker() *repoLocker {
-	return &repoLocker{
+func NewRepoLocker() *RepoLocker {
+	return &RepoLocker{
 		locks: make(map[string]*sync.RWMutex),
 	}
 }
 
 // LockExclusive acquires an exclusive lock for the specified
 // repository. This function blocks until no other locks exist
 // for the repo.
-func (rl *repoLocker) LockExclusive(name string) {
+func (rl *RepoLocker) LockExclusive(name string) {
 	rl.ensureLock(name).Lock()
 }
 
 // Lock acquires a non-exclusive lock for the specified
 // repository. This function blocks until no exclusive
 // locks exist for the repo.
-func (rl *repoLocker) Lock(name string) {
+func (rl *RepoLocker) Lock(name string) {
 	rl.ensureLock(name).RLock()
 }
 
 // UnlockExclusive releases an exclusive lock for the repo.
-func (rl *repoLocker) UnlockExclusive(name string) {
+func (rl *RepoLocker) UnlockExclusive(name string) {
 	rl.ensureLock(name).Unlock()
 }
 
 // Unlock releases a non-exclusive lock for the repo.
-func (rl *repoLocker) Unlock(name string) {
+func (rl *RepoLocker) Unlock(name string) {
 	rl.ensureLock(name).RUnlock()
 }
 
-func (rl *repoLocker) ensureLock(name string) *sync.RWMutex {
+func (rl *RepoLocker) ensureLock(name string) *sync.RWMutex {
 	rl.mu.Lock()
 	defer rl.mu.Unlock()
 

pkg/restic/aws.go → pkg/repository/repoconfig/aws.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 const (
 	// AWS specific environment variable
@@ -23,13 +23,13 @@ const (
 	awsCredentialsFileEnvVar = "AWS_SHARED_CREDENTIALS_FILE"
 )
 
-// getS3ResticEnvVars gets the environment variables that restic
+// GetS3ResticEnvVars gets the environment variables that restic
 // relies on (AWS_PROFILE) based on info in the provided object
 // storage location config map.
-func getS3ResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetS3ResticEnvVars(config map[string]string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		result[awsCredentialsFileEnvVar] = credentialsFile
 	}
 

pkg/restic/aws_test.go → pkg/repository/repoconfig/aws_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"
@@ -55,7 +55,7 @@ func TestGetS3ResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getS3ResticEnvVars(tc.config)
+			actual, err := GetS3ResticEnvVars(tc.config)
 
 			require.NoError(t, err)
 

pkg/restic/azure.go → pkg/repository/repoconfig/azure.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"context"
@@ -131,10 +131,10 @@ func mapLookup(data map[string]string) func(string) string {
 	}
 }
 
-// getAzureResticEnvVars gets the environment variables that restic
+// GetAzureResticEnvVars gets the environment variables that restic
 // relies on (AZURE_ACCOUNT_NAME and AZURE_ACCOUNT_KEY) based
 // on info in the provided object storage location config map.
-func getAzureResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetAzureResticEnvVars(config map[string]string) (map[string]string, error) {
 	storageAccountKey, _, err := getStorageAccountKey(config)
 	if err != nil {
 		return nil, err
@@ -158,7 +158,7 @@ func credentialsFileFromEnv() string {
 // selectCredentialsFile selects the Azure credentials file to use, retrieving it
 // from the given config or falling back to retrieving it from the environment.
 func selectCredentialsFile(config map[string]string) string {
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		return credentialsFile
 	}
 

pkg/restic/azure_test.go → pkg/repository/repoconfig/azure_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"os"

pkg/restic/config.go → pkg/repository/repoconfig/config.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"context"
@@ -37,6 +37,10 @@ const (
 	AWSBackend   BackendType = "velero.io/aws"
 	AzureBackend BackendType = "velero.io/azure"
 	GCPBackend   BackendType = "velero.io/gcp"
+
+	// CredentialsFileKey is the key within a BSL config that is checked to see if
+	// the BSL is using its own credentials, rather than those in the environment
+	CredentialsFileKey = "credentialsFile"
 )
 
 // this func is assigned to a package-level variable so it can be
@@ -55,7 +59,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 		prefix = layout.GetResticDir()
 	}
 
-	backendType := getBackendType(location.Spec.Provider)
+	backendType := GetBackendType(location.Spec.Provider)
 
 	if repoPrefix := location.Spec.Config["resticRepoPrefix"]; repoPrefix != "" {
 		return repoPrefix, nil
@@ -89,7 +93,7 @@ func getRepoPrefix(location *velerov1api.BackupStorageLocation) (string, error)
 	return "", errors.New("restic repository prefix (resticRepoPrefix) not specified in backup storage location's config")
 }
 
-func getBackendType(provider string) BackendType {
+func GetBackendType(provider string) BackendType {
 	if !strings.Contains(provider, "/") {
 		provider = "velero.io/" + provider
 	}

pkg/restic/config_test.go → pkg/repository/repoconfig/config_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"

pkg/restic/gcp.go → pkg/repository/repoconfig/gcp.go

@@ -14,19 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 const (
 	// GCP specific environment variable
 	gcpCredentialsFileEnvVar = "GOOGLE_APPLICATION_CREDENTIALS"
 )
 
-// getGCPResticEnvVars gets the environment variables that restic relies
+// GetGCPResticEnvVars gets the environment variables that restic relies
 // on based on info in the provided object storage location config map.
-func getGCPResticEnvVars(config map[string]string) (map[string]string, error) {
+func GetGCPResticEnvVars(config map[string]string) (map[string]string, error) {
 	result := make(map[string]string)
 
-	if credentialsFile, ok := config[credentialsFileKey]; ok {
+	if credentialsFile, ok := config[CredentialsFileKey]; ok {
 		result[gcpCredentialsFileEnvVar] = credentialsFile
 	}
 

pkg/restic/gcp_test.go → pkg/repository/repoconfig/gcp_test.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package restic
+package repoconfig
 
 import (
 	"testing"
@@ -46,7 +46,7 @@ func TestGetGCPResticEnvVars(t *testing.T) {
 
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
-			actual, err := getGCPResticEnvVars(tc.config)
+			actual, err := GetGCPResticEnvVars(tc.config)
 
 			require.NoError(t, err)