Retry with exponential backoff when fetching artifacts

controllers/helmrelease_controller.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	"github.com/go-logr/logr"
+	"github.com/hashicorp/go-retryablehttp"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chartutil"
 	"helm.sh/helm/v3/pkg/storage/driver"
@@ -71,6 +72,7 @@ import (
 // HelmReleaseReconciler reconciles a HelmRelease object
 type HelmReleaseReconciler struct {
 	client.Client
+	httpClient            *retryablehttp.Client
 	Config                *rest.Config
 	Scheme                *runtime.Scheme
 	requeueDependency     time.Duration
@@ -93,6 +95,16 @@ func (r *HelmReleaseReconciler) SetupWithManager(mgr ctrl.Manager, opts HelmRele
 	}
 
 	r.requeueDependency = opts.DependencyRequeueInterval
+
+	// Configure the retryable http client used for fetching artifacts.
+	// By default it retries 10 times within a 3.5 minutes window.
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryWaitMin = 5 * time.Second
+	httpClient.RetryWaitMax = 30 * time.Second
+	httpClient.RetryMax = opts.HTTPRetry
+	httpClient.Logger = nil
+	r.httpClient = httpClient
+
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&v2.HelmRelease{}, builder.WithPredicates(
 			predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}),
@@ -254,6 +266,7 @@ func (r *HelmReleaseReconciler) reconcile(ctx context.Context, hr v2.HelmRelease
 
 type HelmReleaseReconcilerOptions struct {
 	MaxConcurrentReconciles   int
+	HTTPRetry                 int
 	DependencyRequeueInterval time.Duration
 }
 

controllers/helmrelease_controller_chart.go

@@ -27,6 +27,7 @@ import (
 	"strings"
 
 	"github.com/go-logr/logr"
+	"github.com/hashicorp/go-retryablehttp"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
@@ -109,17 +110,22 @@ func (r *HelmReleaseReconciler) loadHelmChart(source *sourcev1.HelmChart) (*char
 		artifactURL = u.String()
 	}
 
-	res, err := http.Get(artifactURL)
+	req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to create a new request: %w", err)
+	}
+
+	resp, err := r.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to download artifact, error: %w", err)
 	}
-	defer res.Body.Close()
+	defer resp.Body.Close()
 
-	if res.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("artifact '%s' download failed (status code: %s)", source.GetArtifact().URL, res.Status)
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("artifact '%s' download failed (status code: %s)", source.GetArtifact().URL, resp.Status)
 	}
 
-	if _, err = io.Copy(f, res.Body); err != nil {
+	if _, err = io.Copy(f, resp.Body); err != nil {
 		return nil, err
 	}
 

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/fluxcd/pkg/runtime v0.8.3
 	github.com/fluxcd/source-controller/api v0.9.0
 	github.com/go-logr/logr v0.3.0
+	github.com/hashicorp/go-retryablehttp v0.6.8
 	github.com/onsi/ginkgo v1.14.1
 	github.com/onsi/gomega v1.10.2
 	github.com/spf13/pflag v1.0.5

main.go

@@ -62,6 +62,7 @@ func main() {
 		concurrent           int
 		requeueDependency    time.Duration
 		watchAllNamespaces   bool
+		httpRetry            int
 		clientOptions        client.Options
 		logOptions           logger.Options
 	)
@@ -76,6 +77,7 @@ func main() {
 	flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.")
 	flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true,
 		"Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.")
+	flag.IntVar(&httpRetry, "http-retry", 9, "The maximum number of retries when failing to fetch artifacts over HTTP.")
 	flag.CommandLine.MarkDeprecated("log-json", "Please use --log-encoding=json instead.")
 	clientOptions.BindFlags(flag.CommandLine)
 	logOptions.BindFlags(flag.CommandLine)
@@ -130,6 +132,7 @@ func main() {
 	}).SetupWithManager(mgr, controllers.HelmReleaseReconcilerOptions{
 		MaxConcurrentReconciles:   concurrent,
 		DependencyRequeueInterval: requeueDependency,
+		HTTPRetry:                 httpRetry,
 	}); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", v2.HelmReleaseKind)
 		os.Exit(1)