From a8dcafaf2e0d06ce819df5b156e4bf5d6bc3efa5 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Fri, 26 Feb 2021 13:37:45 +0200 Subject: [PATCH] Retry with exponential backoff when fetching artifacts Signed-off-by: Stefan Prodan --- controllers/helmrelease_controller.go | 13 +++++++++++++ controllers/helmrelease_controller_chart.go | 18 ++++++++++++------ go.mod | 1 + main.go | 3 +++ 4 files changed, 29 insertions(+), 6 deletions(-) diff --git a/controllers/helmrelease_controller.go b/controllers/helmrelease_controller.go index 5115ac054..829f0febb 100644 --- a/controllers/helmrelease_controller.go +++ b/controllers/helmrelease_controller.go @@ -24,6 +24,7 @@ import ( "time" "github.com/go-logr/logr" + "github.com/hashicorp/go-retryablehttp" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chartutil" "helm.sh/helm/v3/pkg/storage/driver" @@ -71,6 +72,7 @@ import ( // HelmReleaseReconciler reconciles a HelmRelease object type HelmReleaseReconciler struct { client.Client + httpClient *retryablehttp.Client Config *rest.Config Scheme *runtime.Scheme requeueDependency time.Duration @@ -93,6 +95,16 @@ func (r *HelmReleaseReconciler) SetupWithManager(mgr ctrl.Manager, opts HelmRele } r.requeueDependency = opts.DependencyRequeueInterval + + // Configure the retryable http client used for fetching artifacts. + // By default it retries 10 times within a 3.5 minutes window. + httpClient := retryablehttp.NewClient() + httpClient.RetryWaitMin = 5 * time.Second + httpClient.RetryWaitMax = 30 * time.Second + httpClient.RetryMax = opts.HTTPRetry + httpClient.Logger = nil + r.httpClient = httpClient + return ctrl.NewControllerManagedBy(mgr). For(&v2.HelmRelease{}, builder.WithPredicates( predicate.Or(predicate.GenerationChangedPredicate{}, predicates.ReconcileRequestedPredicate{}), @@ -254,6 +266,7 @@ func (r *HelmReleaseReconciler) reconcile(ctx context.Context, hr v2.HelmRelease type HelmReleaseReconcilerOptions struct { MaxConcurrentReconciles int + HTTPRetry int DependencyRequeueInterval time.Duration } diff --git a/controllers/helmrelease_controller_chart.go b/controllers/helmrelease_controller_chart.go index cf54c2a5a..8d623503b 100644 --- a/controllers/helmrelease_controller_chart.go +++ b/controllers/helmrelease_controller_chart.go @@ -27,6 +27,7 @@ import ( "strings" "github.com/go-logr/logr" + "github.com/hashicorp/go-retryablehttp" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" apierrors "k8s.io/apimachinery/pkg/api/errors" @@ -109,17 +110,22 @@ func (r *HelmReleaseReconciler) loadHelmChart(source *sourcev1.HelmChart) (*char artifactURL = u.String() } - res, err := http.Get(artifactURL) + req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil) if err != nil { - return nil, err + return nil, fmt.Errorf("failed to create a new request: %w", err) + } + + resp, err := r.httpClient.Do(req) + if err != nil { + return nil, fmt.Errorf("failed to download artifact, error: %w", err) } - defer res.Body.Close() + defer resp.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, fmt.Errorf("artifact '%s' download failed (status code: %s)", source.GetArtifact().URL, res.Status) + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("artifact '%s' download failed (status code: %s)", source.GetArtifact().URL, resp.Status) } - if _, err = io.Copy(f, res.Body); err != nil { + if _, err = io.Copy(f, resp.Body); err != nil { return nil, err } diff --git a/go.mod b/go.mod index eaee6f1d7..b875a3836 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/fluxcd/pkg/runtime v0.8.3 github.com/fluxcd/source-controller/api v0.9.0 github.com/go-logr/logr v0.3.0 + github.com/hashicorp/go-retryablehttp v0.6.8 github.com/onsi/ginkgo v1.14.1 github.com/onsi/gomega v1.10.2 github.com/spf13/pflag v1.0.5 diff --git a/main.go b/main.go index 6fa3b0944..ba692fad9 100644 --- a/main.go +++ b/main.go @@ -62,6 +62,7 @@ func main() { concurrent int requeueDependency time.Duration watchAllNamespaces bool + httpRetry int clientOptions client.Options logOptions logger.Options ) @@ -76,6 +77,7 @@ func main() { flag.DurationVar(&requeueDependency, "requeue-dependency", 30*time.Second, "The interval at which failing dependencies are reevaluated.") flag.BoolVar(&watchAllNamespaces, "watch-all-namespaces", true, "Watch for custom resources in all namespaces, if set to false it will only watch the runtime namespace.") + flag.IntVar(&httpRetry, "http-retry", 9, "The maximum number of retries when failing to fetch artifacts over HTTP.") flag.CommandLine.MarkDeprecated("log-json", "Please use --log-encoding=json instead.") clientOptions.BindFlags(flag.CommandLine) logOptions.BindFlags(flag.CommandLine) @@ -130,6 +132,7 @@ func main() { }).SetupWithManager(mgr, controllers.HelmReleaseReconcilerOptions{ MaxConcurrentReconciles: concurrent, DependencyRequeueInterval: requeueDependency, + HTTPRetry: httpRetry, }); err != nil { setupLog.Error(err, "unable to create controller", "controller", v2.HelmReleaseKind) os.Exit(1)