Skip to content

Commit

Permalink
import cycle
Browse files Browse the repository at this point in the history
Signed-off-by: Wenkai Yin(尹文开) <yinw@vmware.com>
  • Loading branch information
ywk253100 committed Aug 29, 2024
1 parent a873e5c commit fbfc60b
Show file tree
Hide file tree
Showing 15 changed files with 331 additions and 348 deletions.
3 changes: 2 additions & 1 deletion pkg/cmd/cli/nodeagent/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ import (
"github.com/vmware-tanzu/velero/pkg/buildinfo"
"github.com/vmware-tanzu/velero/pkg/client"
"github.com/vmware-tanzu/velero/pkg/cmd"
"github.com/vmware-tanzu/velero/pkg/cmd/server/config"
"github.com/vmware-tanzu/velero/pkg/cmd/util/signals"
"github.com/vmware-tanzu/velero/pkg/controller"
"github.com/vmware-tanzu/velero/pkg/datapath"
Expand Down Expand Up @@ -285,7 +286,7 @@ func (s *nodeAgentServer) run() {
credentialGetter, s.nodeName, s.mgr.GetScheme(), s.metrics, s.logger)

if err := pvbReconciler.SetupWithManager(s.mgr); err != nil {
s.logger.Fatal(err, "unable to create controller", "controller", controller.PodVolumeBackup)
s.logger.Fatal(err, "unable to create controller", "controller", config.ControllerPodVolumeBackup)
}

if err = controller.NewPodVolumeRestoreReconciler(s.mgr.GetClient(), s.dataPathMgr, repoEnsurer, credentialGetter, s.logger).SetupWithManager(s.mgr); err != nil {
Expand Down
166 changes: 65 additions & 101 deletions pkg/cmd/server/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,9 @@ import (
"github.com/sirupsen/logrus"
"github.com/spf13/pflag"

// "github.com/vmware-tanzu/velero/pkg/controller"
"github.com/vmware-tanzu/velero/pkg/cmd/util/flag"
"github.com/vmware-tanzu/velero/pkg/podvolume"
"github.com/vmware-tanzu/velero/pkg/repository"

//"github.com/vmware-tanzu/velero/pkg/restore"
"github.com/vmware-tanzu/velero/pkg/uploader"
"github.com/vmware-tanzu/velero/pkg/util/logging"
)
Expand Down Expand Up @@ -48,98 +45,65 @@ const (
// defaultCredentialsDirectory is the path on disk where credential
// files will be written to
defaultCredentialsDirectory = "/tmp/credentials"

ControllerBackup = "backup"
ControllerBackupOperations = "backup-operations"
ControllerBackupDeletion = "backup-deletion"
ControllerBackupFinalizer = "backup-finalizer"
ControllerBackupRepo = "backup-repo"
ControllerBackupStorageLocation = "backup-storage-location"
ControllerBackupSync = "backup-sync"
ControllerDownloadRequest = "download-request"
ControllerGarbageCollection = "gc"
ControllerPodVolumeBackup = "pod-volume-backup"
ControllerPodVolumeRestore = "pod-volume-restore"
ControllerRestore = "restore"
ControllerRestoreOperations = "restore-operations"
ControllerSchedule = "schedule"
ControllerServerStatusRequest = "server-status-request"
ControllerRestoreFinalizer = "restore-finalizer"
)

/*
High priorities:
- Custom Resource Definitions come before Custom Resource so that they can be
restored with their corresponding CRD.
- Namespaces go second because all namespaced resources depend on them.
- Storage Classes are needed to create PVs and PVCs correctly.
- VolumeSnapshotClasses are needed to provision volumes using volumesnapshots
- VolumeSnapshotContents are needed as they contain the handle to the volume snapshot in the
storage provider
- VolumeSnapshots are needed to create PVCs using the VolumeSnapshot as their data source.
- DataUploads need to restore before PVC for Snapshot DataMover to work, because PVC needs the DataUploadResults to create DataDownloads.
- PVs go before PVCs because PVCs depend on them.
- PVCs go before pods or controllers so they can be mounted as volumes.
- Service accounts go before secrets so service account token secrets can be filled automatically.
- Secrets and ConfigMaps go before pods or controllers so they can be mounted
as volumes.
- Limit ranges go before pods or controllers so pods can use them.
- Pods go before controllers so they can be explicitly restored and potentially
have pod volume restores run before controllers adopt the pods.
- Replica sets go before deployments/other controllers so they can be explicitly
restored and be adopted by controllers.
- CAPI ClusterClasses go before Clusters.
- Endpoints go before Services so no new Endpoints will be created
- Services go before Clusters so they can be adopted by AKO-operator and no new Services will be created
for the same clusters
Low priorities:
- Tanzu ClusterBootstraps go last as it can reference any other kind of resources.
- ClusterBootstraps go before CAPI Clusters otherwise a new default ClusterBootstrap object is created for the cluster
- CAPI Clusters come before ClusterResourceSets because failing to do so means the CAPI controller-manager will panic.
Both Clusters and ClusterResourceSets need to come before ClusterResourceSetBinding in order to properly restore workload clusters.
See https://github.com/kubernetes-sigs/cluster-api/issues/4105
*/

// TODO
/*
var defaultRestorePriorities = restore.Priorities{
HighPriorities: []string{
"customresourcedefinitions",
"namespaces",
"storageclasses",
"volumesnapshotclass.snapshot.storage.k8s.io",
"volumesnapshotcontents.snapshot.storage.k8s.io",
"volumesnapshots.snapshot.storage.k8s.io",
"datauploads.velero.io",
"persistentvolumes",
"persistentvolumeclaims",
"serviceaccounts",
"secrets",
"configmaps",
"limitranges",
"pods",
// we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also
// existed in the extensions API group, but we back up replicasets from "apps" so we want
// to ensure that we prioritize restoring from "apps" too, since this is how they're stored
// in the backup.
"replicasets.apps",
"clusterclasses.cluster.x-k8s.io",
"endpoints",
"services",
},
LowPriorities: []string{
"clusterbootstraps.run.tanzu.vmware.com",
"clusters.cluster.x-k8s.io",
"clusterresourcesets.addons.cluster.x-k8s.io",
},
}
*/
var (
// DisableableControllers is a list of controllers that can be disabled
DisableableControllers = []string{
ControllerBackup,
ControllerBackupOperations,
ControllerBackupDeletion,
ControllerBackupFinalizer,
ControllerBackupSync,
ControllerDownloadRequest,
ControllerGarbageCollection,
ControllerBackupRepo,
ControllerRestore,
ControllerRestoreOperations,
ControllerSchedule,
ControllerServerStatusRequest,
ControllerRestoreFinalizer,
}
)

type Config struct {
PluginDir string
MetricsAddress string
DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation
BackupSyncPeriod time.Duration
PodVolumeOperationTimeout time.Duration
ResourceTerminatingTimeout time.Duration
DefaultBackupTTL time.Duration
StoreValidationFrequency time.Duration
DefaultCSISnapshotTimeout time.Duration
DefaultItemOperationTimeout time.Duration
ResourceTimeout time.Duration
// RestoreResourcePriorities restore.Priorities // TODO handle import cycle
PluginDir string
MetricsAddress string
DefaultBackupLocation string // TODO(2.0) Deprecate defaultBackupLocation
BackupSyncPeriod time.Duration
PodVolumeOperationTimeout time.Duration
ResourceTerminatingTimeout time.Duration
DefaultBackupTTL time.Duration
StoreValidationFrequency time.Duration
DefaultCSISnapshotTimeout time.Duration
DefaultItemOperationTimeout time.Duration
ResourceTimeout time.Duration
RestoreResourcePriorities Priorities
DefaultVolumeSnapshotLocations flag.Map
RestoreOnly bool
DisabledControllers []string
ClientQPS float32
ClientBurst int
ClientPageSize int
ProfilerAddress string
LogLevel *logging.LevelFlag // TODO get the value directly?
LogLevel *logging.LevelFlag
LogFormat *logging.FormatFlag
RepoMaintenanceFrequency time.Duration
GarbageCollectionFrequency time.Duration
Expand Down Expand Up @@ -168,21 +132,21 @@ func GetDefaultConfig() *Config {
ResourceTimeout: resourceTimeout,
StoreValidationFrequency: defaultStoreValidationFrequency,
PodVolumeOperationTimeout: defaultPodVolumeOperationTimeout,
//RestoreResourcePriorities: defaultRestorePriorities,
ClientQPS: defaultClientQPS,
ClientBurst: defaultClientBurst,
ClientPageSize: defaultClientPageSize,
ProfilerAddress: defaultProfilerAddress,
ResourceTerminatingTimeout: defaultResourceTerminatingTimeout,
LogLevel: logging.LogLevelFlag(logrus.InfoLevel),
LogFormat: logging.NewFormatFlag(),
DefaultVolumesToFsBackup: podvolume.DefaultVolumesToFsBackup,
UploaderType: uploader.ResticType,
MaxConcurrentK8SConnections: defaultMaxConcurrentK8SConnections,
DefaultSnapshotMoveData: false,
DisableInformerCache: defaultDisableInformerCache,
ScheduleSkipImmediately: false,
CredentialsDirectory: defaultCredentialsDirectory,
RestoreResourcePriorities: defaultRestorePriorities,
ClientQPS: defaultClientQPS,
ClientBurst: defaultClientBurst,
ClientPageSize: defaultClientPageSize,
ProfilerAddress: defaultProfilerAddress,
ResourceTerminatingTimeout: defaultResourceTerminatingTimeout,
LogLevel: logging.LogLevelFlag(logrus.InfoLevel),
LogFormat: logging.NewFormatFlag(),
DefaultVolumesToFsBackup: podvolume.DefaultVolumesToFsBackup,
UploaderType: uploader.ResticType,
MaxConcurrentK8SConnections: defaultMaxConcurrentK8SConnections,
DefaultSnapshotMoveData: false,
DisableInformerCache: defaultDisableInformerCache,
ScheduleSkipImmediately: false,
CredentialsDirectory: defaultCredentialsDirectory,
}

config.MaintenanceCfg = repository.MaintenanceConfig{
Expand All @@ -204,8 +168,8 @@ func (c *Config) BindFlags(flags *pflag.FlagSet) {
flags.DurationVar(&c.BackupSyncPeriod, "backup-sync-period", c.BackupSyncPeriod, "How often to ensure all Velero backups in object storage exist as Backup API objects in the cluster. This is the default sync period if none is explicitly specified for a backup storage location.")
flags.DurationVar(&c.PodVolumeOperationTimeout, "fs-backup-timeout", c.PodVolumeOperationTimeout, "How long pod volume file system backups/restores should be allowed to run before timing out.")
flags.BoolVar(&c.RestoreOnly, "restore-only", c.RestoreOnly, "Run in a mode where only restores are allowed; backups, schedules, and garbage-collection are all disabled. DEPRECATED: this flag will be removed in v2.0. Use read-only backup storage locations instead.")
//flags.StringSliceVar(&c.DisabledControllers, "disable-controllers", c.DisabledControllers, fmt.Sprintf("List of controllers to disable on startup. Valid values are %s", strings.Join(controller.DisableableControllers, ",")))
//flags.Var(&c.RestoreResourcePriorities, "restore-resource-priorities", "Desired order of resource restores, the priority list contains two parts which are split by \"-\" element. The resources before \"-\" element are restored first as high priorities, the resources after \"-\" element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities.")
flags.StringSliceVar(&c.DisabledControllers, "disable-controllers", c.DisabledControllers, fmt.Sprintf("List of controllers to disable on startup. Valid values are %s", strings.Join(DisableableControllers, ",")))
flags.Var(&c.RestoreResourcePriorities, "restore-resource-priorities", "Desired order of resource restores, the priority list contains two parts which are split by \"-\" element. The resources before \"-\" element are restored first as high priorities, the resources after \"-\" element are restored last as low priorities, and any resource not in the list will be restored alphabetically between the high and low priorities.")
flags.StringVar(&c.DefaultBackupLocation, "default-backup-storage-location", c.DefaultBackupLocation, "Name of the default backup storage location. DEPRECATED: this flag will be removed in v2.0. Use \"velero backup-location set --default\" instead.")
flags.DurationVar(&c.StoreValidationFrequency, "store-validation-frequency", c.StoreValidationFrequency, "How often to verify if the storage is valid. Optional. Set this to `0s` to disable sync. Default 1 minute.")
flags.Float32Var(&c.ClientQPS, "client-qps", c.ClientQPS, "Maximum number of requests per second by the server to the Kubernetes API once the burst limit has been reached.")
Expand Down
157 changes: 157 additions & 0 deletions pkg/cmd/server/config/priority.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,157 @@
/*
Copyright The Velero Contributors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
"fmt"
"strings"
)

/*
High priorities:
- Custom Resource Definitions come before Custom Resource so that they can be
restored with their corresponding CRD.
- Namespaces go second because all namespaced resources depend on them.
- Storage Classes are needed to create PVs and PVCs correctly.
- VolumeSnapshotClasses are needed to provision volumes using volumesnapshots
- VolumeSnapshotContents are needed as they contain the handle to the volume snapshot in the
storage provider
- VolumeSnapshots are needed to create PVCs using the VolumeSnapshot as their data source.
- DataUploads need to restore before PVC for Snapshot DataMover to work, because PVC needs the DataUploadResults to create DataDownloads.
- PVs go before PVCs because PVCs depend on them.
- PVCs go before pods or controllers so they can be mounted as volumes.
- Service accounts go before secrets so service account token secrets can be filled automatically.
- Secrets and ConfigMaps go before pods or controllers so they can be mounted
as volumes.
- Limit ranges go before pods or controllers so pods can use them.
- Pods go before controllers so they can be explicitly restored and potentially
have pod volume restores run before controllers adopt the pods.
- Replica sets go before deployments/other controllers so they can be explicitly
restored and be adopted by controllers.
- CAPI ClusterClasses go before Clusters.
- Endpoints go before Services so no new Endpoints will be created
- Services go before Clusters so they can be adopted by AKO-operator and no new Services will be created
for the same clusters
Low priorities:
- Tanzu ClusterBootstraps go last as it can reference any other kind of resources.
- ClusterBootstraps go before CAPI Clusters otherwise a new default ClusterBootstrap object is created for the cluster
- CAPI Clusters come before ClusterResourceSets because failing to do so means the CAPI controller-manager will panic.
Both Clusters and ClusterResourceSets need to come before ClusterResourceSetBinding in order to properly restore workload clusters.
See https://github.com/kubernetes-sigs/cluster-api/issues/4105
*/
var defaultRestorePriorities = Priorities{
HighPriorities: []string{
"customresourcedefinitions",
"namespaces",
"storageclasses",
"volumesnapshotclass.snapshot.storage.k8s.io",
"volumesnapshotcontents.snapshot.storage.k8s.io",
"volumesnapshots.snapshot.storage.k8s.io",
"datauploads.velero.io",
"persistentvolumes",
"persistentvolumeclaims",
"serviceaccounts",
"secrets",
"configmaps",
"limitranges",
"pods",
// we fully qualify replicasets.apps because prior to Kubernetes 1.16, replicasets also
// existed in the extensions API group, but we back up replicasets from "apps" so we want
// to ensure that we prioritize restoring from "apps" too, since this is how they're stored
// in the backup.
"replicasets.apps",
"clusterclasses.cluster.x-k8s.io",
"endpoints",
"services",
},
LowPriorities: []string{
"clusterbootstraps.run.tanzu.vmware.com",
"clusters.cluster.x-k8s.io",
"clusterresourcesets.addons.cluster.x-k8s.io",
},
}

const (
prioritySeparator = "-"
)

// Priorities defines the desired order of resource operations:
// Resources in the HighPriorities list will be handled first
// Resources in the LowPriorities list will be handled last
// Other resources will be handled alphabetically after the high prioritized resources and before the low prioritized resources
type Priorities struct {
HighPriorities []string
LowPriorities []string
}

// String returns a string representation of Priority.
func (p *Priorities) String() string {
priorities := p.HighPriorities
if len(p.LowPriorities) > 0 {
priorities = append(priorities, prioritySeparator)
priorities = append(priorities, p.LowPriorities...)
}
return strings.Join(priorities, ",")
}

// Set parses the provided string to the priority object
func (p *Priorities) Set(s string) error {
if len(s) == 0 {
return nil
}
strs := strings.Split(s, ",")
separatorIndex := -1
for i, str := range strs {
if str == prioritySeparator {
if separatorIndex > -1 {
return fmt.Errorf("multiple priority separator %q found", prioritySeparator)
}
separatorIndex = i
}
}
// has no separator
if separatorIndex == -1 {
p.HighPriorities = strs
return nil
}
// start with separator
if separatorIndex == 0 {
// contain only separator
if len(strs) == 1 {
return nil
}
p.LowPriorities = strs[1:]
return nil
}
// end with separator
if separatorIndex == len(strs)-1 {
p.HighPriorities = strs[:len(strs)-1]
return nil
}

// separator in the middle
p.HighPriorities = strs[:separatorIndex]
p.LowPriorities = strs[separatorIndex+1:]

return nil
}

// Type specifies the flag type
func (p *Priorities) Type() string {
return "stringArray"
}
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/

package restore
package config

import (
"testing"
Expand Down
Loading

0 comments on commit fbfc60b

Please sign in to comment.