Retry deployments (#6801)

* retry wrangler deploy + versions upload
cloudflare · Oct 1, 2024 · 6009bb4 · 6009bb4
1 parent 5f8c584
commit 6009bb4
Show file tree

Hide file tree

Showing 5 changed files with 77 additions and 42 deletions.
diff --git a/.changeset/chilled-trainers-switch.md b/.changeset/chilled-trainers-switch.md
@@ -0,0 +1,5 @@
+---
+"wrangler": minor
+---
+
+feat: implement retries within `wrangler deploy` and `wrangler versions upload` to workaround spotty network connections and service flakes
diff --git a/packages/wrangler/src/__tests__/versions/versions.upload.test.ts b/packages/wrangler/src/__tests__/versions/versions.upload.test.ts
@@ -37,11 +37,16 @@ describe("versions upload", () => {
 			)
 		);
 	}
-	function mockUploadVersion(has_preview: boolean) {
+	function mockUploadVersion(has_preview: boolean, flakeCount = 1) {
 		msw.use(
 			http.post(
 				`*/accounts/:accountId/workers/scripts/:scriptName/versions`,
 				({ params }) => {
+					if (flakeCount > 0) {
+						flakeCount--;
+						return HttpResponse.error();
+					}
+
 					expect(params.scriptName).toEqual("test-worker");
 
 					return HttpResponse.json(
@@ -53,8 +58,7 @@ describe("versions upload", () => {
 							},
 						})
 					);
-				},
-				{ once: true }
+				}
 			)
 		);
 	}

diff --git a/packages/wrangler/src/deploy/deploy.ts b/packages/wrangler/src/deploy/deploy.ts
@@ -46,6 +46,7 @@ import {
 	maybeRetrieveFileSourceMap,
 } from "../sourcemap";
 import triggersDeploy from "../triggers/deploy";
+import { retryOnError } from "../utils/retry";
 import {
 	createDeployment,
 	patchNonVersionedScriptSettings,
@@ -814,13 +815,15 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 				// If we're using the new APIs, first upload the version
 				if (canUseNewVersionsDeploymentsApi) {
 					// Upload new version
-					const versionResult = await fetchResult<ApiVersion>(
-						`/accounts/${accountId}/workers/scripts/${scriptName}/versions`,
-						{
-							method: "POST",
-							body: createWorkerUploadForm(worker),
-							headers: await getMetricsUsageHeaders(config.send_metrics),
-						}
+					const versionResult = await retryOnError(async () =>
+						fetchResult<ApiVersion>(
+							`/accounts/${accountId}/workers/scripts/${scriptName}/versions`,
+							{
+								method: "POST",
+								body: createWorkerUploadForm(worker),
+								headers: await getMetricsUsageHeaders(config.send_metrics),
+							}
+						)
 					);
 
 					// Deploy new version to 100%
@@ -852,27 +855,29 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 						startup_time_ms: versionResult.startup_time_ms,
 					};
 				} else {
-					result = await fetchResult<{
-						available_on_subdomain: boolean;
-						id: string | null;
-						etag: string | null;
-						pipeline_hash: string | null;
-						mutable_pipeline_id: string | null;
-						deployment_id: string | null;
-						startup_time_ms: number;
-					}>(
-						workerUrl,
-						{
-							method: "PUT",
-							body: createWorkerUploadForm(worker),
-							headers: await getMetricsUsageHeaders(config.send_metrics),
-						},
-						new URLSearchParams({
-							include_subdomain_availability: "true",
-							// pass excludeScript so the whole body of the
-							// script doesn't get included in the response
-							excludeScript: "true",
-						})
+					result = await retryOnError(async () =>
+						fetchResult<{
+							available_on_subdomain: boolean;
+							id: string | null;
+							etag: string | null;
+							pipeline_hash: string | null;
+							mutable_pipeline_id: string | null;
+							deployment_id: string | null;
+							startup_time_ms: number;
+						}>(
+							workerUrl,
+							{
+								method: "PUT",
+								body: createWorkerUploadForm(worker),
+								headers: await getMetricsUsageHeaders(config.send_metrics),
+							},
+							new URLSearchParams({
+								include_subdomain_availability: "true",
+								// pass excludeScript so the whole body of the
+								// script doesn't get included in the response
+								excludeScript: "true",
+							})
+						)
 					);
 				}
 

diff --git a/packages/wrangler/src/utils/retry.ts b/packages/wrangler/src/utils/retry.ts
@@ -0,0 +1,18 @@
+import { setTimeout } from "node:timers/promises";
+
+export async function retryOnError<T>(
+	action: () => T | Promise<T>,
+	backoff = 2_000,
+	attempts = 3
+): Promise<T> {
+	try {
+		return await action();
+	} catch (err) {
+		if (attempts <= 1) {
+			throw err;
+		}
+
+		await setTimeout(backoff);
+		return retryOnError(action, backoff, attempts - 1);
+	}
+}
diff --git a/packages/wrangler/src/versions/upload.ts b/packages/wrangler/src/versions/upload.ts
@@ -35,6 +35,7 @@ import {
 	getSourceMappedString,
 	maybeRetrieveFileSourceMap,
 } from "../sourcemap";
+import { retryOnError } from "../utils/retry";
 import type { AssetsOptions } from "../assets";
 import type { Config } from "../config";
 import type { Rule } from "../config/environment";
@@ -466,17 +467,19 @@ See https://developers.cloudflare.com/workers/platform/compatibility-dates for m
 			try {
 				const body = createWorkerUploadForm(worker);
 
-				const result = await fetchResult<{
-					id: string;
-					startup_time_ms: number;
-					metadata: {
-						has_preview: boolean;
-					};
-				}>(`${workerUrl}/versions`, {
-					method: "POST",
-					body,
-					headers: await getMetricsUsageHeaders(config.send_metrics),
-				});
+				const result = await retryOnError(async () =>
+					fetchResult<{
+						id: string;
+						startup_time_ms: number;
+						metadata: {
+							has_preview: boolean;
+						};
+					}>(`${workerUrl}/versions`, {
+						method: "POST",
+						body,
+						headers: await getMetricsUsageHeaders(config.send_metrics),
+					})
+				);
 
 				logger.log("Worker Startup Time:", result.startup_time_ms, "ms");
 				bindingsPrinted = true;