diff --git a/packages/components/src/color-picker/component.tsx b/packages/components/src/color-picker/component.tsx
index 0a3d381bdc875..bd9ee1fd49465 100644
--- a/packages/components/src/color-picker/component.tsx
+++ b/packages/components/src/color-picker/component.tsx
@@ -11,7 +11,6 @@ import namesPlugin from 'colord/plugins/names';
  */
 import { useState, useMemo } from '@wordpress/element';
 import { settings } from '@wordpress/icons';
-import { useDebounce } from '@wordpress/compose';
 import { __ } from '@wordpress/i18n';
 
 /**
@@ -33,7 +32,6 @@ import {
 import { ColorDisplay } from './color-display';
 import { ColorInput } from './color-input';
 import { Picker } from './picker';
-import { useControlledValue } from '../utils/hooks';
 
 import type { ColorType } from './types';
 
@@ -59,32 +57,23 @@ const ColorPicker = (
 ) => {
 	const {
 		enableAlpha = false,
-		color: colorProp,
+		color,
 		onChange,
-		defaultValue,
+		defaultValue = '#fff',
 		copyFormat,
 		...divProps
 	} = useContextSystem( props, 'ColorPicker' );
 
-	const [ color, setColor ] = useControlledValue( {
-		onChange,
-		value: colorProp,
-		defaultValue,
-	} );
-
 	// Use a safe default value for the color and remove the possibility of `undefined`.
 	const safeColordColor = useMemo( () => {
-		return color ? colord( color ) : colord( '#fff' );
-	}, [ color ] );
-
-	// Debounce to prevent rapid changes from conflicting with one another.
-	const debouncedSetColor = useDebounce( setColor );
+		return color ? colord( color ) : colord( defaultValue );
+	}, [ color, defaultValue ] );
 
 	const handleChange = useCallback(
 		( nextValue: Colord ) => {
-			debouncedSetColor( nextValue.toHex() );
+			onChange( nextValue.toHex() );
 		},
-		[ debouncedSetColor ]
+		[ onChange ]
 	);
 
 	const [ showInputs, setShowInputs ] = useState< boolean >( false );
diff --git a/packages/components/src/color-picker/picker.tsx b/packages/components/src/color-picker/picker.tsx
index 3304112001e25..f48050a468d9e 100644
--- a/packages/components/src/color-picker/picker.tsx
+++ b/packages/components/src/color-picker/picker.tsx
@@ -1,7 +1,7 @@
 /**
  * External dependencies
  */
-import { HslColorPicker, HslaColorPicker } from 'react-colorful';
+import { RgbStringColorPicker, RgbaStringColorPicker } from 'react-colorful';
 import { colord, Colord } from 'colord';
 
 /**
@@ -15,12 +15,14 @@ interface PickerProps {
 }
 
 export const Picker = ( { color, enableAlpha, onChange }: PickerProps ) => {
-	const Component = enableAlpha ? HslaColorPicker : HslColorPicker;
-	const hslColor = useMemo( () => color.toHsl(), [ color ] );
+	const Component = enableAlpha
+		? RgbaStringColorPicker
+		: RgbStringColorPicker;
+	const rgbColor = useMemo( () => color.toRgbString(), [ color ] );
 
 	return (
 		<Component
-			color={ hslColor }
+			color={ rgbColor }
 			onChange={ ( nextColor ) => {
 				onChange( colord( nextColor ) );
 			} }
diff --git a/packages/components/src/color-picker/test/index.js b/packages/components/src/color-picker/test/index.js
index 19f8f2130f005..34a5198c23607 100644
--- a/packages/components/src/color-picker/test/index.js
+++ b/packages/components/src/color-picker/test/index.js
@@ -41,12 +41,6 @@ function moveReactColorfulSlider( sliderElement, from, to ) {
 	fireEvent( sliderElement, new FakeMouseEvent( 'mousemove', to ) );
 }
 
-const sleep = ( ms ) => {
-	const promise = new Promise( ( resolve ) => setTimeout( resolve, ms ) );
-	jest.advanceTimersByTime( ms + 1 );
-	return promise;
-};
-
 const hslaMatcher = expect.objectContaining( {
 	h: expect.any( Number ),
 	s: expect.any( Number ),
@@ -93,9 +87,6 @@ describe( 'ColorPicker', () => {
 				{ pageX: 10, pageY: 10 }
 			);
 
-			// `onChange` is debounced so we need to sleep for at least 1ms before checking that onChange was called
-			await sleep( 1 );
-
 			expect( onChangeComplete ).toHaveBeenCalledWith(
 				legacyColorMatcher
 			);
@@ -117,9 +108,6 @@ describe( 'ColorPicker', () => {
 			{ pageX: 10, pageY: 10 }
 		);
 
-		// `onChange` is debounced so we need to sleep for at least 1ms before checking that onChange was called
-		await sleep( 1 );
-
 		expect( onChange ).toHaveBeenCalledWith(
 			expect.stringMatching( /^#([a-fA-F0-9]{8})$/ )
 		);
@@ -150,9 +138,6 @@ describe( 'ColorPicker', () => {
 			{ pageX: 10, pageY: 10 }
 		);
 
-		// `onChange` is debounced so we need to sleep for at least 1ms before checking that onChange was called
-		await sleep( 1 );
-
 		expect( onChange ).toHaveBeenCalledWith(
 			expect.stringMatching( /^#([a-fA-F0-9]{6})$/ )
 		);