ZTS: Replace MD5 and SHA256 wit XXH128

For data integrity checks as done in ZTS, the verification for
unintended data corruption with xxhash128 should be a lot faster
and perfectly usable.

Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
Closes #16577

.github/workflows/scripts/qemu-3-deps.sh

@@ -16,7 +16,7 @@ function archlinux() {
   sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \
     fio gdb inetutils jq less linux linux-headers lsscsi nfs-utils parted \
     pax perf python-packaging python-setuptools qemu-guest-agent ksh samba \
-    sysstat rng-tools rsync wget
+    sysstat rng-tools rsync wget xxhash
   echo "##[endgroup]"
 }
 
@@ -38,7 +38,7 @@ function debian() {
     lsscsi nfs-kernel-server pamtester parted python3 python3-all-dev \
     python3-cffi python3-dev python3-distlib python3-packaging \
     python3-setuptools python3-sphinx qemu-guest-agent rng-tools rpm2cpio \
-    rsync samba sysstat uuid-dev watchdog wget xfslibs-dev zlib1g-dev
+    rsync samba sysstat uuid-dev watchdog wget xfslibs-dev  xxhash zlib1g-dev
   echo "##[endgroup]"
 }
 
@@ -48,8 +48,7 @@ function freebsd() {
   echo "##[group]Install Development Tools"
   sudo pkg install -y autoconf automake autotools base64 checkbashisms fio \
     gdb gettext gettext-runtime git gmake gsed jq ksh93 lcov libtool lscpu \
-    pkgconf python python3 pamtester pamtester qemu-guest-agent rsync \
-    sysutils/coreutils
+    pkgconf python python3 pamtester pamtester qemu-guest-agent rsync xxhash
   sudo pkg install -xy \
     '^samba4[[:digit:]]+$' \
     '^py3[[:digit:]]+-cffi$' \
@@ -76,7 +75,7 @@ function rhel() {
     lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \
     python3 python3-cffi python3-devel python3-packaging kernel-devel \
     python3-setuptools qemu-guest-agent rng-tools rpcgen rpm-build rsync \
-    samba sysstat systemd watchdog wget xfsprogs-devel zlib-devel
+    samba sysstat systemd watchdog wget xfsprogs-devel xxhash zlib-devel
   echo "##[endgroup]"
 }
 

tests/zfs-tests/include/commands.cfg

@@ -100,7 +100,8 @@ export SYSTEM_FILES_COMMON='awk
     uniq
     vmstat
     wc
-    xargs'
+    xargs
+    xxh128sum'
 
 export SYSTEM_FILES_FREEBSD='chflags
     compress
@@ -112,13 +113,11 @@ export SYSTEM_FILES_FREEBSD='chflags
     jexec
     jls
     lsextattr
-    md5
     mdconfig
     newfs
     pw
     rmextattr
     setextattr
-    sha256
     showmount
     swapctl
     sysctl
@@ -146,7 +145,6 @@ export SYSTEM_FILES_LINUX='attr
     lscpu
     lsmod
     lsscsi
-    md5sum
     mkswap
     modprobe
     mountpoint
@@ -156,7 +154,6 @@ export SYSTEM_FILES_LINUX='attr
     perf
     setfattr
     setpriv
-    sha256sum
     udevadm
     unshare
     useradd

tests/zfs-tests/include/libtest.shlib

@@ -3455,35 +3455,26 @@ function tunable_exists
 }
 
 #
-# Compute MD5 digest for given file or stdin if no file given.
+# Compute xxh128sum for given file or stdin if no file given.
 # Note: file path must not contain spaces
 #
-function md5digest
+function xxh128digest
 {
-	openssl md5 -r $1 | awk '{print $1}'
+	xxh128sum $1 | awk '{print $1}'
 }
 
 #
-# Compare the MD5 digest of two files.
+# Compare the xxhash128 digest of two files.
 #
-function cmp_md5s {
+function cmp_xxh128 {
 	typeset file1=$1
 	typeset file2=$2
 
-	typeset sum1=$(md5digest $file1)
-	typeset sum2=$(md5digest $file2)
+	typeset sum1=$(xxh128digest $file1)
+	typeset sum2=$(xxh128digest $file2)
 	test "$sum1" = "$sum2"
 }
 
-#
-# Compute SHA256 digest for given file or stdin if no file given.
-# Note: file path must not contain spaces
-#
-function sha256digest
-{
-	openssl sha256 -r $1 | awk '{print $1}'
-}
-
 function new_fs #<args>
 {
 	case "$UNAME" in

tests/zfs-tests/tests/functional/bclone/bclone_common.kshlib

@@ -77,7 +77,7 @@ function test_file_integrity
     typeset -r clone=$2
     typeset -r filesize=$3
 
-    typeset -r clone_checksum=$(sha256digest $clone)
+    typeset -r clone_checksum=$(xxh128digest $clone)
 
     if [[ $original_checksum != $clone_checksum ]]; then
         log_fail "Clone $clone is corrupted with file size $filesize"
@@ -171,7 +171,7 @@ function bclone_test
         dsize=0
     fi
 
-    typeset -r original_checksum=$(sha256digest $original)
+    typeset -r original_checksum=$(xxh128digest $original)
 
     sync_pool $TESTPOOL
 

tests/zfs-tests/tests/functional/bclone/bclone_corner_cases.kshlib

@@ -32,14 +32,14 @@ function first_half_checksum
 {
     typeset -r file=$1
 
-    dd if=$file bs=$HALFRECORDSIZE count=1 2>/dev/null | sha256digest
+    dd if=$file bs=$HALFRECORDSIZE count=1 2>/dev/null | xxh128digest
 }
 
 function second_half_checksum
 {
     typeset -r file=$1
 
-    dd if=$file bs=$HALFRECORDSIZE count=1 skip=1 2>/dev/null | sha256digest
+    dd if=$file bs=$HALFRECORDSIZE count=1 skip=1 2>/dev/null | xxh128digest
 }
 
 function bclone_corner_cases_init
@@ -66,7 +66,7 @@ function bclone_corner_cases_init
     export SECOND_HALF_ORIG0_CHECKSUM=$(second_half_checksum $ORIG0)
     export SECOND_HALF_ORIG1_CHECKSUM=$(second_half_checksum $ORIG1)
     export SECOND_HALF_ORIG2_CHECKSUM=$(second_half_checksum $ORIG2)
-    export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 2>/dev/null | sha256digest)
+    export ZEROS_CHECKSUM=$(dd if=/dev/zero bs=$HALFRECORDSIZE count=1 2>/dev/null | xxh128digest)
     export FIRST_HALF_CHECKSUM=""
     export SECOND_HALF_CHECKSUM=""
 }

tests/zfs-tests/tests/functional/block_cloning/block_cloning.kshlib

@@ -28,8 +28,8 @@
 
 function have_same_content
 {
-	typeset hash1=$(md5digest $1)
-	typeset hash2=$(md5digest $2)
+	typeset hash1=$(xxh128digest $1)
+	typeset hash2=$(xxh128digest $2)
 
 	log_must [ "$hash1" = "$hash2" ]
 }

tests/zfs-tests/tests/functional/block_cloning/block_cloning_cross_enc_dataset.ksh

@@ -110,9 +110,9 @@ log_note "Cloning entire file with copy_file_range across different enc" \
 clone_and_check "file" "clone" $DS1 $DS2 "" true
 log_note "check if the file is still readable and the same after" \
     "unmount and key unload, shouldn't fail"
-typeset hash1=$(md5digest "/$DS1/file")
+typeset hash1=$(xxh128digest "/$DS1/file")
 log_must zfs umount $DS1 && zfs unload-key $DS1
-typeset hash2=$(md5digest "/$DS2/clone")
+typeset hash2=$(xxh128digest "/$DS2/clone")
 log_must [ "$hash1" = "$hash2" ]
 
 cleanup_enc
@@ -144,12 +144,12 @@ log_must sync_pool $TESTPOOL
 log_must rm -f "/$DS1/file" "/$DS2/file"
 log_must sync_pool $TESTPOOL
 clone_and_check "file" "clone" "$DS2" "$DS1" "" true "s1"
-typeset hash1=$(md5digest "/$DS1/.zfs/snapshot/s1/file")
+typeset hash1=$(xxh128digest "/$DS1/.zfs/snapshot/s1/file")
 log_note "destroy the snapshot and check if the file is still readable and" \
     "has the same content"
 log_must zfs destroy -r $DS2@s1
 log_must sync_pool $TESTPOOL
-typeset hash2=$(md5digest "/$DS1/file")
+typeset hash2=$(xxh128digest "/$DS1/file")
 log_must [ "$hash1" = "$hash2" ]
 
 cleanup_enc

tests/zfs-tests/tests/functional/cli_root/zdb/zdb_backup.ksh

@@ -47,8 +47,8 @@ sync_pool $TESTPOOL
 log_must eval "zfs send -ecL $snap > $tmpfile.1"
 log_must eval "zdb -B $TESTPOOL/$objsetid ecL > $tmpfile.2"
 
-typeset sum1=$(md5digest $tmpfile.1)
-typeset sum2=$(md5digest $tmpfile.2)
+typeset sum1=$(xxh128digest $tmpfile.1)
+typeset sum2=$(xxh128digest $tmpfile.2)
 
 log_must test "$sum1" = "$sum2"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_compressed_corrective.ksh

@@ -73,7 +73,7 @@ function test_corrective_recv
 	log_must zpool status -v $TESTPOOL
 	log_mustnot eval "zpool status -v $TESTPOOL | \
 	    grep \"Permanent errors have been detected\""
-	typeset cksum=$(md5digest $file)
+	typeset cksum=$(xxh128digest $file)
 	[[ "$cksum" == "$checksum" ]] || \
 		log_fail "Checksums differ ($cksum != $checksum)"
 }
@@ -95,7 +95,7 @@ log_must zfs create -o primarycache=none \
 
 log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
 log_must eval "echo 'aaaaaaaa' >> "$file
-typeset checksum=$(md5digest $file)
+typeset checksum=$(xxh128digest $file)
 
 log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_corrective.ksh

@@ -72,7 +72,7 @@ function test_corrective_recv
 	log_must zpool status -v $TESTPOOL
 	log_mustnot eval "zpool status -v $TESTPOOL | \
 	    grep \"Permanent errors have been detected\""
-	typeset cksum=$(md5digest $file)
+	typeset cksum=$(xxh128digest $file)
 	[[ "$cksum" == "$checksum" ]] || \
 		log_fail "Checksums differ ($cksum != $checksum)"
 }
@@ -94,7 +94,7 @@ log_must zfs create -o primarycache=none \
 
 log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
 log_must eval "echo 'aaaaaaaa' >> "$file
-typeset checksum=$(md5digest $file)
+typeset checksum=$(xxh128digest $file)
 
 log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
 
@@ -177,7 +177,7 @@ log_must zpool scrub -w $TESTPOOL
 log_must zpool status -v $TESTPOOL
 log_mustnot eval "zpool status -v $TESTPOOL | \
     grep \"Permanent errors have been detected\""
-typeset cksum=$(md5digest $file)
+typeset cksum=$(xxh128digest $file)
 [[ "$cksum" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_from_encrypted.ksh

@@ -59,7 +59,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 	"-o keyformat=passphrase $TESTPOOL/$TESTFS2"
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS2/$TESTFILE0
-typeset checksum=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
+typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 
 log_must zfs snapshot $snap
 
@@ -69,14 +69,14 @@ log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS1/c1"
 crypt=$(get_prop encryption $TESTPOOL/$TESTFS1/c1)
 [[ "$crypt" == "off" ]] || log_fail "Received unencrypted stream as encrypted"
 
-typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
+typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 
 log_note "Verify ZFS can receive into an encrypted child"
 log_must eval "zfs send $snap | zfs receive $TESTPOOL/$TESTFS2/c1"
 
-typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
+typeset cksum2=$(xxh128digest /$TESTPOOL/$TESTFS2/c1/$TESTFILE0)
 [[ "$cksum2" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum2 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_from_zstd.ksh

@@ -60,7 +60,7 @@ log_must zfs create -o compress=zstd-$random_level $TESTPOOL/$TESTFS1
 # Make a 5kb compressible file
 log_must eval cat $src_data $src_data $src_data $src_data $src_data \
     "> /$TESTPOOL/$TESTFS1/$TESTFILE0"
-typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
+typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
 
 log_must zfs snapshot $snap
 
@@ -79,7 +79,7 @@ log_note "ZSTD src: size=$zstd_size1 version=$zstd_version1 level=$zstd_level1"
 log_note "Verify ZFS can receive the ZSTD compressed stream"
 log_must eval "zfs send -ec $snap | zfs receive $TESTPOOL/$TESTFS2"
 
-typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
+typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_large_block_corrective.ksh

@@ -73,7 +73,7 @@ function test_corrective_recv
 	log_must zpool status -v $TESTPOOL
 	log_mustnot eval "zpool status -v $TESTPOOL | \
 	    grep \"Permanent errors have been detected\""
-	typeset cksum=$(md5digest $file)
+	typeset cksum=$(xxh128digest $file)
 	[[ "$cksum" == "$checksum" ]] || \
 		log_fail "Checksums differ ($cksum != $checksum)"
 }
@@ -96,7 +96,7 @@ log_must zfs create -o recordsize=1m -o primarycache=none \
 
 log_must dd if=/dev/urandom of=$file bs=1024 count=1024 oflag=sync
 log_must eval "echo 'aaaaaaaa' >> "$file
-typeset checksum=$(md5digest $file)
+typeset checksum=$(xxh128digest $file)
 
 log_must zfs snapshot $TESTPOOL/$TESTFS1@snap1
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw.ksh

@@ -61,7 +61,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 	"-o keyformat=passphrase $TESTPOOL/$TESTFS1"
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
-typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
+typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
 
 log_must zfs snapshot $snap
 
@@ -74,7 +74,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS2)
 
 log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS2"
 
-typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
+typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 
@@ -85,7 +85,7 @@ keystatus=$(get_prop keystatus $TESTPOOL/$TESTFS1/c1)
 	log_fail "Expected keystatus unavailable, got $keystatus"
 
 log_must eval "echo $passphrase | zfs mount -l $TESTPOOL/$TESTFS1/c1"
-typeset cksum2=$(md5digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
+typeset cksum2=$(xxh128digest /$TESTPOOL/$TESTFS1/c1/$TESTFILE0)
 [[ "$cksum2" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum2 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zfs_receive/zfs_receive_raw_incremental.ksh

@@ -69,7 +69,7 @@ log_must eval "echo $passphrase | zfs create -o encryption=on" \
 log_must zfs snapshot $snap1
 
 log_must mkfile 1M /$TESTPOOL/$TESTFS1/$TESTFILE0
-typeset checksum=$(md5digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
+typeset checksum=$(xxh128digest /$TESTPOOL/$TESTFS1/$TESTFILE0)
 
 log_must zfs snapshot $snap2
 
@@ -89,7 +89,7 @@ log_must zfs unload-key $TESTPOOL/$TESTFS2
 log_must eval "zfs receive $TESTPOOL/$TESTFS2 < $ibackup"
 log_must eval "echo $passphrase2 | zfs mount -l $TESTPOOL/$TESTFS2"
 
-typeset cksum1=$(md5digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
+typeset cksum1=$(xxh128digest /$TESTPOOL/$TESTFS2/$TESTFILE0)
 [[ "$cksum1" == "$checksum" ]] || \
 	log_fail "Checksums differ ($cksum1 != $checksum)"
 

tests/zfs-tests/tests/functional/cli_root/zpool_import/import_cachefile_shared_device.ksh

@@ -50,7 +50,7 @@ function dev_checksum
 
 	log_note "Compute checksum of '$dev'"
 
-	md5digest $dev ||
+	xxh128digest $dev ||
 		log_fail "Failed to compute checksum of '$dev'"
 }
 

tests/zfs-tests/tests/functional/cli_root/zpool_import/import_devices_missing.ksh

@@ -79,11 +79,11 @@ function test_devices_missing
 	log_must set_spa_load_verify_data 0
 	log_must zpool import -o readonly=on -d $DEVICE_DIR $TESTPOOL1
 
-	log_must verify_data_md5sums $MD5FILE
+	log_must verify_data_hashsums $MD5FILE
 
 	log_note "Try reading second batch of data, make sure pool doesn't" \
 	    "get suspended."
-	verify_data_md5sums $MD5FILE >/dev/null 2>&1
+	verify_data_hashsums $MD5FILE >/dev/null 2>&1
 
 	log_must_busy zpool export $TESTPOOL1
 
@@ -95,8 +95,8 @@ function test_devices_missing
 	log_must set_zfs_max_missing_tvds 0
 	log_must zpool import -d $DEVICE_DIR $TESTPOOL1
 
-	log_must verify_data_md5sums $MD5FILE
-	log_must verify_data_md5sums $MD5FILE2
+	log_must verify_data_hashsums $MD5FILE
+	log_must verify_data_hashsums $MD5FILE2
 
 	# Cleanup
 	log_must zpool destroy $TESTPOOL1