diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go
index 392fe11953c75..a85b9d85cb170 100644
--- a/expression/builtin_cast.go
+++ b/expression/builtin_cast.go
@@ -284,12 +284,7 @@ func (c *castAsStringFunctionClass) getFunction(ctx sessionctx.Context, args []E
 		return nil, err
 	}
 	bf.tp = c.tp
-	if args[0].GetType().Hybrid() || IsBinaryLiteral(args[0]) {
-		// When cast from binary to some other charsets, we should check if the binary is valid or not.
-		// so we build a from_binary function to do this check.
-		ft := args[0].GetType().Clone()
-		ft.Charset, ft.Collate = c.tp.Charset, c.tp.Collate
-		bf.args[0] = BuildFromBinaryFunction(ctx, args[0], ft)
+	if args[0].GetType().Hybrid() {
 		sig = &builtinCastStringAsStringSig{bf}
 		sig.setPbCode(tipb.ScalarFuncSig_CastStringAsString)
 		return sig, nil
@@ -318,6 +313,9 @@ func (c *castAsStringFunctionClass) getFunction(ctx sessionctx.Context, args []E
 		sig = &builtinCastJSONAsStringSig{bf}
 		sig.setPbCode(tipb.ScalarFuncSig_CastJsonAsString)
 	case types.ETString:
+		// When cast from binary to some other charsets, we should check if the binary is valid or not.
+		// so we build a from_binary function to do this check.
+		bf.args[0] = HandleBinaryLiteral(ctx, args[0], &ExprCollation{Charset: c.tp.Charset, Collation: c.tp.Collate}, c.funcName)
 		sig = &builtinCastStringAsStringSig{bf}
 		sig.setPbCode(tipb.ScalarFuncSig_CastStringAsString)
 	default:
diff --git a/expression/builtin_cast_test.go b/expression/builtin_cast_test.go
index 0d60f9d8b6e3d..43958afe78ec8 100644
--- a/expression/builtin_cast_test.go
+++ b/expression/builtin_cast_test.go
@@ -1420,6 +1420,9 @@ func TestWrapWithCastAsString(t *testing.T) {
 			require.Equal(t, c.ret, res)
 		}
 	}
+
+	expr := BuildCastFunction(ctx, &Constant{RetType: types.NewFieldType(mysql.TypeEnum)}, types.NewFieldType(mysql.TypeVarString))
+	require.NotContains(t, expr.String(), "to_binary")
 }
 
 func TestWrapWithCastAsJSON(t *testing.T) {
diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go
index a24ed138eb578..5c0ea98f752a5 100644
--- a/expression/builtin_convert_charset.go
+++ b/expression/builtin_convert_charset.go
@@ -282,7 +282,7 @@ var convertActionMap = map[funcProp][]string{
 		ast.Replace, ast.Rpad, ast.SubstringIndex, ast.Trim,
 		/* operators */
 		ast.GE, ast.LE, ast.GT, ast.LT, ast.EQ, ast.NE, ast.NullEQ, ast.If, ast.Ifnull, ast.In,
-		ast.Case,
+		ast.Case, ast.Cast,
 		/* string comparing */
 		ast.Like, ast.Strcmp,
 		/* regex */