Block cloning tests.

The test mostly focus on testing various corner cases.
The tests take a long time to run, so for the common.run runfile
we randomly select a hundred tests.
To run all the bclone tests, bclone.run runfile should be used.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Pawel Jakub Dawidek <pawel@dawidek.net>
Closes #15631

tests/Makefile.am

@@ -16,6 +16,7 @@ dist_scripts_test_runner_include_DATA = \
 
 scripts_runfilesdir = $(datadir)/$(PACKAGE)/runfiles
 dist_scripts_runfiles_DATA = \
+	%D%/runfiles/bclone.run \
 	%D%/runfiles/common.run \
 	%D%/runfiles/freebsd.run \
 	%D%/runfiles/linux.run \

tests/runfiles/bclone.run

@@ -0,0 +1,46 @@
+#
+# This file and its contents are supplied under the terms of the
+# Common Development and Distribution License ("CDDL"), version 1.0.
+# You may only use this file in accordance with the terms of version
+# 1.0 of the CDDL.
+#
+# A full copy of the text of the CDDL should have accompanied this
+# source.  A copy of the CDDL is also available via the Internet at
+# http://www.illumos.org/license/CDDL.
+#
+# This run file contains all of the common functional tests.  When
+# adding a new test consider also adding it to the sanity.run file
+# if the new test runs to completion in only a few seconds.
+#
+# Approximate run time: 5 hours
+#
+
+[DEFAULT]
+pre = setup
+quiet = False
+pre_user = root
+user = root
+timeout = 28800
+post_user = root
+post = cleanup
+failsafe_user = root
+failsafe = callbacks/zfs_failsafe
+outputdir = /var/tmp/test_results
+tags = ['bclone']
+
+[tests/functional/bclone]
+tests = ['bclone_crossfs_corner_cases',
+    'bclone_crossfs_data',
+    'bclone_crossfs_embedded',
+    'bclone_crossfs_hole',
+    'bclone_diffprops_all',
+    'bclone_diffprops_checksum',
+    'bclone_diffprops_compress',
+    'bclone_diffprops_copies',
+    'bclone_diffprops_recordsize',
+    'bclone_prop_sync',
+    'bclone_samefs_corner_cases',
+    'bclone_samefs_data',
+    'bclone_samefs_embedded',
+    'bclone_samefs_hole']
+tags = ['bclone']

tests/runfiles/common.run

@@ -53,6 +53,24 @@ tags = ['functional', 'arc']
 tests = ['atime_001_pos', 'atime_002_neg', 'root_atime_off', 'root_atime_on']
 tags = ['functional', 'atime']
 
+[tests/functional/bclone]
+tests = ['bclone_crossfs_corner_cases_limited',
+    'bclone_crossfs_data',
+    'bclone_crossfs_embedded',
+    'bclone_crossfs_hole',
+    'bclone_diffprops_all',
+    'bclone_diffprops_checksum',
+    'bclone_diffprops_compress',
+    'bclone_diffprops_copies',
+    'bclone_diffprops_recordsize',
+    'bclone_prop_sync',
+    'bclone_samefs_corner_cases_limited',
+    'bclone_samefs_data',
+    'bclone_samefs_embedded',
+    'bclone_samefs_hole']
+tags = ['functional', 'bclone']
+timeout = 7200
+
 [tests/functional/bootfs]
 tests = ['bootfs_001_pos', 'bootfs_002_neg', 'bootfs_003_pos',
     'bootfs_004_neg', 'bootfs_005_neg', 'bootfs_006_pos', 'bootfs_007_pos',

tests/test-runner/bin/zts-report.py.in

@@ -263,13 +263,50 @@ if sys.platform.startswith('freebsd'):
         'cli_root/zpool_import/zpool_import_012_pos': ['FAIL', known_reason],
         'delegate/zfs_allow_003_pos': ['FAIL', known_reason],
         'inheritance/inherit_001_pos': ['FAIL', 11829],
-        'resilver/resilver_restart_001': ['FAIL', known_reason],
         'pool_checkpoint/checkpoint_big_rewind': ['FAIL', 12622],
         'pool_checkpoint/checkpoint_indirect': ['FAIL', 12623],
+        'resilver/resilver_restart_001': ['FAIL', known_reason],
         'snapshot/snapshot_002_pos': ['FAIL', '14831'],
     })
 elif sys.platform.startswith('linux'):
     maybe.update({
+        'bclone/bclone_crossfs_corner_cases': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_crossfs_corner_cases_limited':
+            ['SKIP', cfr_cross_reason],
+        'bclone/bclone_crossfs_data': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_crossfs_embedded': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_crossfs_hole': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_diffprops_all': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_diffprops_checksum': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_diffprops_compress': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_diffprops_copies': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_diffprops_recordsize': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_prop_sync': ['SKIP', cfr_cross_reason],
+        'bclone/bclone_samefs_corner_cases': ['SKIP', cfr_reason],
+        'bclone/bclone_samefs_corner_cases_limited': ['SKIP', cfr_reason],
+        'bclone/bclone_samefs_data': ['SKIP', cfr_reason],
+        'bclone/bclone_samefs_embedded': ['SKIP', cfr_reason],
+        'bclone/bclone_samefs_hole': ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_copyfilerange':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_copyfilerange_cross_dataset':
+            ['SKIP', cfr_cross_reason],
+        'block_cloning/block_cloning_copyfilerange_fallback':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_copyfilerange_fallback_same_txg':
+            ['SKIP', cfr_cross_reason],
+        'block_cloning/block_cloning_copyfilerange_partial':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_cross_enc_dataset':
+            ['SKIP', cfr_cross_reason],
+        'block_cloning/block_cloning_disabled_copyfilerange':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_lwb_buffer_overflow':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_replay':
+            ['SKIP', cfr_reason],
+        'block_cloning/block_cloning_replay_encrypted':
+            ['SKIP', cfr_reason],
         'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason],
         'cli_root/zpool_reopen/zpool_reopen_003_pos': ['FAIL', known_reason],
         'fault/auto_online_002_pos': ['FAIL', 11889],
@@ -278,41 +315,21 @@ elif sys.platform.startswith('linux'):
         'fault/auto_spare_multiple': ['FAIL', 11889],
         'fault/auto_spare_shared': ['FAIL', 11889],
         'fault/decompress_fault': ['FAIL', 11889],
+        'idmap_mount/idmap_mount_001': ['SKIP', idmap_reason],
+        'idmap_mount/idmap_mount_002': ['SKIP', idmap_reason],
+        'idmap_mount/idmap_mount_003': ['SKIP', idmap_reason],
+        'idmap_mount/idmap_mount_004': ['SKIP', idmap_reason],
+        'idmap_mount/idmap_mount_005': ['SKIP', idmap_reason],
         'io/io_uring': ['SKIP', 'io_uring support required'],
         'limits/filesystem_limit': ['SKIP', known_reason],
         'limits/snapshot_limit': ['SKIP', known_reason],
         'mmp/mmp_active_import': ['FAIL', known_reason],
         'mmp/mmp_exported_import': ['FAIL', known_reason],
         'mmp/mmp_inactive_import': ['FAIL', known_reason],
-        'zvol/zvol_misc/zvol_misc_snapdev': ['FAIL', 12621],
-        'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason],
         'zvol/zvol_misc/zvol_misc_fua': ['SKIP', 14872],
+        'zvol/zvol_misc/zvol_misc_snapdev': ['FAIL', 12621],
         'zvol/zvol_misc/zvol_misc_trim': ['SKIP', 14872],
-        'idmap_mount/idmap_mount_001': ['SKIP', idmap_reason],
-        'idmap_mount/idmap_mount_002': ['SKIP', idmap_reason],
-        'idmap_mount/idmap_mount_003': ['SKIP', idmap_reason],
-        'idmap_mount/idmap_mount_004': ['SKIP', idmap_reason],
-        'idmap_mount/idmap_mount_005': ['SKIP', idmap_reason],
-        'block_cloning/block_cloning_disabled_copyfilerange':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_copyfilerange':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_copyfilerange_partial':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_copyfilerange_fallback':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_replay':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_replay_encrypted':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_lwb_buffer_overflow':
-            ['SKIP', cfr_reason],
-        'block_cloning/block_cloning_copyfilerange_cross_dataset':
-            ['SKIP', cfr_cross_reason],
-        'block_cloning/block_cloning_copyfilerange_fallback_same_txg':
-            ['SKIP', cfr_cross_reason],
-        'block_cloning/block_cloning_cross_enc_dataset':
-            ['SKIP', cfr_cross_reason],
+        'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason],
     })
 
 # Not all Github actions runners have scsi_debug module, so we may skip

tests/zfs-tests/cmd/Makefile.am

@@ -2,6 +2,7 @@ scripts_zfs_tests_bindir = $(datadir)/$(PACKAGE)/zfs-tests/bin
 
 
 scripts_zfs_tests_bin_PROGRAMS  = %D%/chg_usr_exec
+scripts_zfs_tests_bin_PROGRAMS += %D%/clonefile
 scripts_zfs_tests_bin_PROGRAMS += %D%/cp_files
 scripts_zfs_tests_bin_PROGRAMS += %D%/ctime
 scripts_zfs_tests_bin_PROGRAMS += %D%/dir_rd_update
@@ -119,7 +120,6 @@ scripts_zfs_tests_bin_PROGRAMS += %D%/renameat2
 scripts_zfs_tests_bin_PROGRAMS += %D%/xattrtest
 scripts_zfs_tests_bin_PROGRAMS += %D%/zed_fd_spill-zedlet
 scripts_zfs_tests_bin_PROGRAMS += %D%/idmap_util
-scripts_zfs_tests_bin_PROGRAMS += %D%/clonefile
 
 %C%_idmap_util_LDADD = libspl.la
 

tests/zfs-tests/cmd/clonefile.c

@@ -59,6 +59,10 @@
 #endif
 #endif /* __NR_copy_file_range */
 
+#ifdef __FreeBSD__
+#define	loff_t	off_t
+#endif
+
 ssize_t
 copy_file_range(int, loff_t *, int, loff_t *, size_t, unsigned int)
     __attribute__((weak));
@@ -140,7 +144,7 @@ usage(void)
 	    "  FICLONERANGE:\n"
 	    "    clonefile -r <src> <dst> <soff> <doff> <len>\n"
 	    "  copy_file_range:\n"
-	    "    clonefile -f <src> <dst> <soff> <doff> <len>\n"
+	    "    clonefile -f <src> <dst> [<soff> <doff> <len | \"all\">]\n"
 	    "  FIDEDUPERANGE:\n"
 	    "    clonefile -d <src> <dst> <soff> <doff> <len>\n");
 	return (1);
@@ -179,13 +183,29 @@ main(int argc, char **argv)
 		}
 	}
 
-	if (mode == CF_MODE_NONE || (argc-optind) < 2 ||
-	    (mode != CF_MODE_CLONE && (argc-optind) < 5))
-		return (usage());
+	switch (mode) {
+		case CF_MODE_NONE:
+			return (usage());
+		case CF_MODE_CLONE:
+			if ((argc-optind) != 2)
+				return (usage());
+			break;
+		case CF_MODE_CLONERANGE:
+		case CF_MODE_DEDUPERANGE:
+			if ((argc-optind) != 5)
+				return (usage());
+			break;
+		case CF_MODE_COPYFILERANGE:
+			if ((argc-optind) != 2 && (argc-optind) != 5)
+				return (usage());
+			break;
+		default:
+			abort();
+	}
 
 	loff_t soff = 0, doff = 0;
-	size_t len = 0;
-	if (mode != CF_MODE_CLONE) {
+	size_t len = SSIZE_MAX;
+	if ((argc-optind) == 5) {
 		soff = strtoull(argv[optind+2], NULL, 10);
 		if (soff == ULLONG_MAX) {
 			fprintf(stderr, "invalid source offset");
@@ -196,10 +216,15 @@ main(int argc, char **argv)
 			fprintf(stderr, "invalid dest offset");
 			return (1);
 		}
-		len = strtoull(argv[optind+4], NULL, 10);
-		if (len == ULLONG_MAX) {
-			fprintf(stderr, "invalid length");
-			return (1);
+		if (mode == CF_MODE_COPYFILERANGE &&
+		    strcmp(argv[optind+4], "all") == 0) {
+			len = SSIZE_MAX;
+		} else {
+			len = strtoull(argv[optind+4], NULL, 10);
+			if (len == ULLONG_MAX) {
+				fprintf(stderr, "invalid length");
+				return (1);
+			}
 		}
 	}
 
@@ -237,13 +262,15 @@ main(int argc, char **argv)
 			abort();
 	}
 
-	off_t spos = lseek(sfd, 0, SEEK_CUR);
-	off_t slen = lseek(sfd, 0, SEEK_END);
-	off_t dpos = lseek(dfd, 0, SEEK_CUR);
-	off_t dlen = lseek(dfd, 0, SEEK_END);
+	if (!quiet) {
+		off_t spos = lseek(sfd, 0, SEEK_CUR);
+		off_t slen = lseek(sfd, 0, SEEK_END);
+		off_t dpos = lseek(dfd, 0, SEEK_CUR);
+		off_t dlen = lseek(dfd, 0, SEEK_END);
 
-	fprintf(stderr, "file offsets: src=%lu/%lu; dst=%lu/%lu\n", spos, slen,
-	    dpos, dlen);
+		fprintf(stderr, "file offsets: src=%lu/%lu; dst=%lu/%lu\n",
+		    spos, slen, dpos, dlen);
+	}
 
 	close(dfd);
 	close(sfd);
@@ -254,7 +281,8 @@ main(int argc, char **argv)
 int
 do_clone(int sfd, int dfd)
 {
-	fprintf(stderr, "using FICLONE\n");
+	if (!quiet)
+		fprintf(stderr, "using FICLONE\n");
 	int err = ioctl(dfd, CF_FICLONE, sfd);
 	if (err < 0) {
 		fprintf(stderr, "ioctl(FICLONE): %s\n", strerror(errno));
@@ -266,7 +294,8 @@ do_clone(int sfd, int dfd)
 int
 do_clonerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using FICLONERANGE\n");
+	if (!quiet)
+		fprintf(stderr, "using FICLONERANGE\n");
 	cf_file_clone_range_t fcr = {
 		.src_fd = sfd,
 		.src_offset = soff,
@@ -284,12 +313,22 @@ do_clonerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 int
 do_copyfilerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using copy_file_range\n");
+	if (!quiet)
+		fprintf(stderr, "using copy_file_range\n");
 	ssize_t copied = cf_copy_file_range(sfd, &soff, dfd, &doff, len, 0);
 	if (copied < 0) {
 		fprintf(stderr, "copy_file_range: %s\n", strerror(errno));
 		return (1);
 	}
+	if (len == SSIZE_MAX) {
+		struct stat sb;
+
+		if (fstat(sfd, &sb) < 0) {
+			fprintf(stderr, "fstat(sfd): %s\n", strerror(errno));
+			return (1);
+		}
+		len = sb.st_size;
+	}
 	if (copied != len) {
 		fprintf(stderr, "copy_file_range: copied less than requested: "
 		    "requested=%lu; copied=%lu\n", len, copied);
@@ -301,7 +340,8 @@ do_copyfilerange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 int
 do_deduperange(int sfd, int dfd, loff_t soff, loff_t doff, size_t len)
 {
-	fprintf(stderr, "using FIDEDUPERANGE\n");
+	if (!quiet)
+		fprintf(stderr, "using FIDEDUPERANGE\n");
 
 	char buf[sizeof (cf_file_dedupe_range_t)+
 	    sizeof (cf_file_dedupe_range_info_t)] = {0};

tests/zfs-tests/include/commands.cfg

@@ -98,7 +98,8 @@ export SYSTEM_FILES_COMMON='awk
     uname
     uniq
     vmstat
-    wc'
+    wc
+    xargs'
 
 export SYSTEM_FILES_FREEBSD='chflags
     compress

tests/zfs-tests/include/math.shlib

@@ -123,10 +123,21 @@ function verify_ne # <a> <b> <type>
 #
 # $1 lower bound
 # $2 upper bound
+# [$3 how many]
 function random_int_between
 {
 	typeset -i min=$1
 	typeset -i max=$2
+	typeset -i count
+	typeset -i i
 
-	echo $(( (RANDOM % (max - min + 1)) + min ))
+	if [[ -z "$3" ]]; then
+		count=1
+	else
+		count=$3
+	fi
+
+	for (( i = 0; i < $count; i++ )); do
+		echo $(( (RANDOM % (max - min + 1)) + min ))
+	done
 }