diff --git a/fs/ext4/inline.c b/fs/ext4/inline.c
index 6d253edebf9fc4..0d42c5f7de5446 100644
--- a/fs/ext4/inline.c
+++ b/fs/ext4/inline.c
@@ -1122,8 +1122,8 @@ static int ext4_update_inline_dir(handle_t *handle, struct inode *dir,
 }
 
 static void ext4_restore_inline_data(handle_t *handle, struct inode *inode,
-				     struct ext4_iloc *iloc,
-				     void *buf, int inline_size)
+				     struct ext4_iloc *iloc, void *buf,
+				     int inline_size, bool has_data)
 {
 	int ret;
 
@@ -1135,7 +1135,8 @@ static void ext4_restore_inline_data(handle_t *handle, struct inode *inode,
 		return;
 	}
 	ext4_write_inline_data(inode, iloc, buf, 0, inline_size);
-	ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
+	if (has_data)
+		ext4_set_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
 }
 
 static int ext4_finish_convert_inline_dir(handle_t *handle,
@@ -1191,6 +1192,7 @@ static int ext4_convert_inline_data_nolock(handle_t *handle,
 	struct buffer_head *data_bh = NULL;
 	struct ext4_map_blocks map;
 	int inline_size;
+	bool has_data;
 
 	inline_size = ext4_get_inline_size(inode);
 	buf = kmalloc(inline_size, GFP_NOFS);
@@ -1219,6 +1221,8 @@ static int ext4_convert_inline_data_nolock(handle_t *handle,
 	if (error)
 		goto out;
 
+	has_data = !!ext4_test_inode_state(inode, EXT4_STATE_MAY_INLINE_DATA);
+
 	map.m_lblk = 0;
 	map.m_len = 1;
 	map.m_flags = 0;
@@ -1259,7 +1263,8 @@ static int ext4_convert_inline_data_nolock(handle_t *handle,
 	unlock_buffer(data_bh);
 out_restore:
 	if (error)
-		ext4_restore_inline_data(handle, inode, iloc, buf, inline_size);
+		ext4_restore_inline_data(handle, inode, iloc, buf,
+					 inline_size, has_data);
 
 out:
 	brelse(data_bh);