diff --git a/packages/block-library/src/image/image.js b/packages/block-library/src/image/image.js
index 48ea75f1333de..4b4879fba7e58 100644
--- a/packages/block-library/src/image/image.js
+++ b/packages/block-library/src/image/image.js
@@ -165,7 +165,14 @@ export default function Image( {
 		}
 	}, [ isSelected ] );
 
-	useParentAttributes( image, context, inheritedAttributes, setAttributes );
+	const currentAttributes = { linkDestination, linkTarget, href };
+	useParentAttributes(
+		image,
+		currentAttributes,
+		context,
+		inheritedAttributes,
+		setAttributes
+	);
 
 	// If an image is externally hosted, try to fetch the image data. This may
 	// fail if the image host doesn't allow CORS with the domain. If it works,
@@ -209,8 +216,8 @@ export default function Image( {
 
 	function onSetHref( props ) {
 		if (
-			inheritedAttributes.linkDestination ||
-			inheritedAttributes.linkTarget
+			inheritedAttributes?.linkDestination ||
+			inheritedAttributes?.linkTarget
 		) {
 			setAttributes( {
 				inheritedAttributes: {
@@ -251,7 +258,7 @@ export default function Image( {
 	}
 
 	function updateImage( newSizeSlug ) {
-		if ( inheritedAttributes.sizeSlug ) {
+		if ( inheritedAttributes?.sizeSlug ) {
 			setAttributes( {
 				inheritedAttributes: {
 					...inheritedAttributes,
diff --git a/packages/block-library/src/image/use-parent-attributes.js b/packages/block-library/src/image/use-parent-attributes.js
index bbbf4099e7196..80e79fcb12d91 100644
--- a/packages/block-library/src/image/use-parent-attributes.js
+++ b/packages/block-library/src/image/use-parent-attributes.js
@@ -13,12 +13,14 @@ import { getUrl, getImageSizeAttributes } from './utils';
  * from changed values supplied by parent block context.
  *
  * @param {Object} image                  Image.
+ * @param {Object} currentAttributes      Current values for inheritable attributes
  * @param {Object} context                Parent block context.
  * @param {Object} inheritedAttributes    Image block attribute that indicate which attributes are inherited from parent.
  * @param {Function} setAttributes        Image block setAttributes prop.
  */
 export default function useParentAttributes(
 	image,
+	currentAttributes,
 	context,
 	inheritedAttributes,
 	setAttributes
@@ -35,6 +37,21 @@ export default function useParentAttributes(
 			setAttributes( {
 				inheritedAttributes: undefined,
 			} );
+			return;
+		}
+		if ( isGrouped && ! inheritedAttributes ) {
+			// Check current Image attributes to make sure we don't overwrite an custom
+			// links and targets. from Image blocks dragged into a new parent block with inheritable
+			// attributes
+			const { linkDestination, linkTarget, href } = currentAttributes;
+			setAttributes( {
+				inheritedAttributes: {
+					linkDestination:
+						linkDestination !== 'none' || href ? false : true,
+					linkTarget: linkTarget ? false : true,
+					sizeSlug: true,
+				},
+			} );
 		}
 	}, [ isGrouped, inheritedAttributes ] );
 
@@ -62,7 +79,7 @@ export default function useParentAttributes(
 	}, [ parentLinkTarget ] );
 
 	useEffect( () => {
-		if ( ! isGrouped || ! inheritedAttributes.sizeSlug ) {
+		if ( ! isGrouped || ! inheritedAttributes?.sizeSlug ) {
 			return;
 		}