Make sure to add filter and view file when needed.

packages/block-library/src/image/index.php

@@ -38,16 +38,6 @@ function render_block_core_image( $attributes, $content, $block ) {
 	$view_js_file_handle = 'wp-block-image-view';
 	$script_handles      = $block->block_type->view_script_handles;
 
-	/*
-	 * Remove the filter and the JavaScript view file if previously added by
-	 * other Image blocks.
-	 */
-	remove_filter( 'render_block_core/image', 'block_core_image_render_lightbox', 15 );
-	// If the script is not needed, and it is still in the `view_script_handles`, remove it.
-	if ( in_array( $view_js_file_handle, $script_handles, true ) ) {
-		$block->block_type->view_script_handles = array_diff( $script_handles, array( $view_js_file_handle ) );
-	}
-
 	/*
 	 * If the lightbox is enabled and the image is not linked, add the filter
 	 * and the JavaScript view file.
@@ -73,6 +63,16 @@ function render_block_core_image( $attributes, $content, $block ) {
 		 * rendered changes, or if a new kind of filter is introduced.
 		 */
 		add_filter( 'render_block_core/image', 'block_core_image_render_lightbox', 15, 2 );
+	} else {
+		/*
+		* Remove the filter and the JavaScript view file if previously added by
+		* other Image blocks.
+		*/
+		remove_filter( 'render_block_core/image', 'block_core_image_render_lightbox', 15 );
+		// If the script is not needed, and it is still in the `view_script_handles`, remove it.
+		if ( in_array( $view_js_file_handle, $script_handles, true ) ) {
+			$block->block_type->view_script_handles = array_diff( $script_handles, array( $view_js_file_handle ) );
+		}
 	}
 
 	return $processor->get_updated_html();