Merge pull request #7619 from allenxu404/post_restore_hook_enhancement

Wait for results of restore exec hook executions in Finalizing phase instead of InProgress phase

changelogs/unreleased/7619-allenxu404

@@ -0,0 +1 @@
+Wait for results of restore exec hook executions in Finalizing phase instead of InProgress phase

internal/hook/hook_tracker.go

@@ -26,8 +26,8 @@ const (
 	HookSourceSpec       = "spec"
 )
 
-// hookTrackerKey identifies a backup/restore hook
-type hookTrackerKey struct {
+// hookKey identifies a backup/restore hook
+type hookKey struct {
 	// PodNamespace indicates the namespace of pod where hooks are executed.
 	// For hooks specified in the backup/restore spec, this field is the namespace of an applicable pod.
 	// For hooks specified in pod annotation, this field is the namespace of pod where hooks are annotated.
@@ -48,37 +48,46 @@ type hookTrackerKey struct {
 	container string
 }
 
-// hookTrackerVal records the execution status of a specific hook.
-// hookTrackerVal is extensible to accommodate additional fields as needs develop.
-type hookTrackerVal struct {
+// hookStatus records the execution status of a specific hook.
+// hookStatus is extensible to accommodate additional fields as needs develop.
+type hookStatus struct {
 	// HookFailed indicates if hook failed to execute.
 	hookFailed bool
 	// hookExecuted indicates if hook already execute.
 	hookExecuted bool
 }
 
-// HookTracker tracks all hooks' execution status
+// HookTracker tracks all hooks' execution status in a single backup/restore.
 type HookTracker struct {
-	lock    *sync.RWMutex
-	tracker map[hookTrackerKey]hookTrackerVal
+	lock *sync.RWMutex
+	// tracker records all hook info for a single backup/restore.
+	tracker map[hookKey]hookStatus
+	// hookAttemptedCnt indicates the number of attempted hooks.
+	hookAttemptedCnt int
+	// hookFailedCnt indicates the number of failed hooks.
+	hookFailedCnt int
+	// HookExecutedCnt indicates the number of executed hooks.
+	hookExecutedCnt int
+	// hookErrs records hook execution errors if any.
+	hookErrs []HookErrInfo
 }
 
-// NewHookTracker creates a hookTracker.
+// NewHookTracker creates a hookTracker instance.
 func NewHookTracker() *HookTracker {
 	return &HookTracker{
 		lock:    &sync.RWMutex{},
-		tracker: make(map[hookTrackerKey]hookTrackerVal),
+		tracker: make(map[hookKey]hookStatus),
 	}
 }
 
-// Add adds a hook to the tracker
+// Add adds a hook to the hook tracker
 // Add must precede the Record for each individual hook.
 // In other words, a hook must be added to the tracker before its execution result is recorded.
 func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName string, hookPhase hookPhase) {
 	ht.lock.Lock()
 	defer ht.lock.Unlock()
 
-	key := hookTrackerKey{
+	key := hookKey{
 		podNamespace: podNamespace,
 		podName:      podName,
 		hookSource:   source,
@@ -88,21 +97,22 @@ func (ht *HookTracker) Add(podNamespace, podName, container, source, hookName st
 	}
 
 	if _, ok := ht.tracker[key]; !ok {
-		ht.tracker[key] = hookTrackerVal{
+		ht.tracker[key] = hookStatus{
 			hookFailed:   false,
 			hookExecuted: false,
 		}
+		ht.hookAttemptedCnt++
 	}
 }
 
 // Record records the hook's execution status
 // Add must precede the Record for each individual hook.
 // In other words, a hook must be added to the tracker before its execution result is recorded.
-func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase hookPhase, hookFailed bool) error {
+func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName string, hookPhase hookPhase, hookFailed bool, hookErr error) error {
 	ht.lock.Lock()
 	defer ht.lock.Unlock()
 
-	key := hookTrackerKey{
+	key := hookKey{
 		podNamespace: podNamespace,
 		podName:      podName,
 		hookSource:   source,
@@ -111,38 +121,125 @@ func (ht *HookTracker) Record(podNamespace, podName, container, source, hookName
 		hookName:     hookName,
 	}
 
-	var err error
-	if _, ok := ht.tracker[key]; ok {
-		ht.tracker[key] = hookTrackerVal{
+	if _, ok := ht.tracker[key]; !ok {
+		return fmt.Errorf("hook not exist in hook tracker, hook: %+v", key)
+	}
+
+	if !ht.tracker[key].hookExecuted {
+		ht.tracker[key] = hookStatus{
 			hookFailed:   hookFailed,
 			hookExecuted: true,
 		}
-	} else {
-		err = fmt.Errorf("hook not exist in hooks tracker, hook key: %v", key)
+		ht.hookExecutedCnt++
+		if hookFailed {
+			ht.hookFailedCnt++
+			ht.hookErrs = append(ht.hookErrs, HookErrInfo{Namespace: key.podNamespace, Err: hookErr})
+		}
 	}
-	return err
+	return nil
 }
 
-// Stat calculates the number of attempted hooks and failed hooks
-func (ht *HookTracker) Stat() (hookAttemptedCnt int, hookFailed int) {
+// Stat returns the number of attempted hooks and failed hooks
+func (ht *HookTracker) Stat() (hookAttemptedCnt int, hookFailedCnt int) {
 	ht.lock.RLock()
 	defer ht.lock.RUnlock()
 
-	for _, hookInfo := range ht.tracker {
-		if hookInfo.hookExecuted {
-			hookAttemptedCnt++
-			if hookInfo.hookFailed {
-				hookFailed++
-			}
-		}
-	}
-	return
+	return ht.hookAttemptedCnt, ht.hookFailedCnt
+}
+
+// IsComplete returns whether the execution of all hooks has finished or not
+func (ht *HookTracker) IsComplete() bool {
+	ht.lock.RLock()
+	defer ht.lock.RUnlock()
+
+	return ht.hookAttemptedCnt == ht.hookExecutedCnt
 }
 
-// GetTracker gets the tracker inside HookTracker
-func (ht *HookTracker) GetTracker() map[hookTrackerKey]hookTrackerVal {
+// HooksErr returns hook execution errors
+func (ht *HookTracker) HookErrs() []HookErrInfo {
 	ht.lock.RLock()
 	defer ht.lock.RUnlock()
 
-	return ht.tracker
+	return ht.hookErrs
+}
+
+// MultiHookTrackers tracks all hooks' execution status for multiple backups/restores.
+type MultiHookTracker struct {
+	lock *sync.RWMutex
+	// trackers is a map that uses the backup/restore name as the key and stores a HookTracker as value.
+	trackers map[string]*HookTracker
+}
+
+// NewMultiHookTracker creates a multiHookTracker instance.
+func NewMultiHookTracker() *MultiHookTracker {
+	return &MultiHookTracker{
+		lock:     &sync.RWMutex{},
+		trackers: make(map[string]*HookTracker),
+	}
+}
+
+// Add adds a backup/restore hook to the tracker
+func (mht *MultiHookTracker) Add(name, podNamespace, podName, container, source, hookName string, hookPhase hookPhase) {
+	mht.lock.Lock()
+	defer mht.lock.Unlock()
+
+	if _, ok := mht.trackers[name]; !ok {
+		mht.trackers[name] = NewHookTracker()
+	}
+	mht.trackers[name].Add(podNamespace, podName, container, source, hookName, hookPhase)
+}
+
+// Record records a backup/restore hook execution status
+func (mht *MultiHookTracker) Record(name, podNamespace, podName, container, source, hookName string, hookPhase hookPhase, hookFailed bool, hookErr error) error {
+	mht.lock.RLock()
+	defer mht.lock.RUnlock()
+
+	var err error
+	if _, ok := mht.trackers[name]; ok {
+		err = mht.trackers[name].Record(podNamespace, podName, container, source, hookName, hookPhase, hookFailed, hookErr)
+	} else {
+		err = fmt.Errorf("the backup/restore not exist in hook tracker, backup/restore name: %s", name)
+	}
+	return err
+}
+
+// Stat returns the number of attempted hooks and failed hooks for a particular backup/restore
+func (mht *MultiHookTracker) Stat(name string) (hookAttemptedCnt int, hookFailedCnt int) {
+	mht.lock.RLock()
+	defer mht.lock.RUnlock()
+
+	if _, ok := mht.trackers[name]; ok {
+		return mht.trackers[name].Stat()
+	}
+	return
+}
+
+// Delete removes the hook data for a particular backup/restore
+func (mht *MultiHookTracker) Delete(name string) {
+	mht.lock.Lock()
+	defer mht.lock.Unlock()
+
+	delete(mht.trackers, name)
+}
+
+// IsComplete returns whether the execution of all hooks for a particular backup/restore has finished or not
+func (mht *MultiHookTracker) IsComplete(name string) bool {
+	mht.lock.RLock()
+	defer mht.lock.RUnlock()
+
+	if _, ok := mht.trackers[name]; ok {
+		return mht.trackers[name].IsComplete()
+	}
+	return true
+}
+
+// HooksErr returns hook execution errors for a particular backup/restore
+func (mht *MultiHookTracker) HookErrs(name string) []HookErrInfo {
+	mht.lock.RLock()
+	defer mht.lock.RUnlock()
+
+	if _, ok := mht.trackers[name]; ok {
+		return mht.trackers[name].HookErrs()
+	}
+	return nil
 }