Add retry logic to packagekit's signtool (#697)

Add retry logic to `signtool`. This works around the transient error I get when I'm running large rebuild jobs

pkg/packagekit/authenticode/authenticode_windows.go

@@ -12,6 +12,7 @@ import (
 	"context"
 	"os/exec"
 	"strings"
+	"time"
 
 	"github.com/go-kit/kit/log"
 	"github.com/go-kit/kit/log/level"
@@ -20,6 +21,12 @@ import (
 	"go.opencensus.io/trace"
 )
 
+const (
+	maxRetries         = 5
+	retryDelay         = 30 * time.Second
+	timeoutErrorString = "The specified timestamp server either could not be reached"
+)
+
 // Sign uses signtool to add authenticode signatures. It supports
 // optional arguments to allow cert specification
 func Sign(ctx context.Context, file string, opts ...SigntoolOpt) error {
@@ -81,6 +88,8 @@ func Sign(ctx context.Context, file string, opts ...SigntoolOpt) error {
 
 // signtoolSign appends some arguments and execs
 func (so *signtoolOptions) signtoolSign(ctx context.Context, file string, args ...string) error {
+	logger := log.With(ctxlog.FromContext(ctx), "caller", log.DefaultCaller)
+
 	ctx, span := trace.StartSpan(ctx, "signtoolSign")
 	defer span.End()
 
@@ -92,8 +101,32 @@ func (so *signtoolOptions) signtoolSign(ctx context.Context, file string, args .
 
 	args = append(args, file)
 
-	if _, _, err := so.execOut(ctx, so.signtoolPath, args...); err != nil {
+	// The timestamp servers timeout sometimes. So we
+	// implement a retry logic here.
+	attempt := 0
+	for {
+		attempt = attempt + 1
+		_, stderr, err := so.execOut(ctx, so.signtoolPath, args...)
+		if err == nil {
+			return nil
+		}
+
+		if attempt > maxRetries {
+			return errors.Wrapf(err, "Reached max number of retries. %d is too many", attempt)
+		}
+
+		if !strings.Contains(stderr, timeoutErrorString) {
+			level.Debug(logger).Log(
+				"msg", "signtool got a retryable error. Sleeping and will try again",
+				"attempt", attempt,
+				"err", err,
+			)
+			time.Sleep(retryDelay)
+			continue
+		}
+
+		// Fallthrough to catch errors unrelated to timeouts
 		return errors.Wrap(err, "calling signtool")
 	}
-	return nil
+	return errors.New("How did you get here? Some logic bug")
 }