diff --git a/packages/block-library/src/image/image.js b/packages/block-library/src/image/image.js
index 4b4879fba7e58..85985660c3d65 100644
--- a/packages/block-library/src/image/image.js
+++ b/packages/block-library/src/image/image.js
@@ -222,9 +222,10 @@ export default function Image( {
 			setAttributes( {
 				inheritedAttributes: {
 					...inheritedAttributes,
-					linkDestination: props.linkDestination
-						? false
-						: inheritedAttributes.linkDestination,
+					linkDestination:
+						props.linkDestination || props.href
+							? false
+							: inheritedAttributes.linkDestination,
 					linkTarget:
 						props.linkTarget || props.linkTarget !== linkTarget
 							? false