From b21f6b7b1d65927cc1ea32e2c44062c759b002b1 Mon Sep 17 00:00:00 2001
From: Tony Hutter <hutter2@llnl.gov>
Date: Thu, 20 Jun 2024 10:32:40 -0700
Subject: [PATCH] Use QEMU for tests on Linux and FreeBSD

-----------------------------------------------------------------
Do not merge - this is my testing version
-Tony Hutter

Requires-builders: none
----------------------------------------------------------------

This commit adds functional tests for these systems:
- AlmaLinux 8, AlmaLinux 9
- ArchLinux
- CentOS Stream 8, CentOS Stream 9
- Fedora 38, Fedora 39
- Debian 11, Debian 12
- FreeBSD 13, FreeBSD 14, FreeBSD 15
- Ubuntu 22.04, Ubuntu 24.04

Workflow for each operating system:
- install QEMU on the github runner
- download cloud image for this system
- start and init that image via cloud-init
- install deps, build openzfs, load the module
- do the functional testings, hopefully < 5h

Signed-off-by: Tino Reichardt <milky-zfs@mcmilk.de>
---
 .github/workflows/scripts/README.md           |  10 +
 .github/workflows/scripts/generate-summary.sh | 258 ++++++++++++------
 .github/workflows/scripts/qemu-1-setup.sh     |  42 +++
 .github/workflows/scripts/qemu-2-start.sh     | 160 +++++++++++
 .github/workflows/scripts/qemu-3-deps.sh      | 174 ++++++++++++
 .github/workflows/scripts/qemu-4-build.sh     | 148 ++++++++++
 .github/workflows/scripts/qemu-5-setup.sh     |  85 ++++++
 .github/workflows/scripts/qemu-6-tests.sh     | 127 +++++++++
 .github/workflows/scripts/qemu-7-reports.sh   |  38 +++
 .../workflows/scripts/setup-dependencies.sh   |  88 ------
 .github/workflows/scripts/setup-functional.sh |  24 --
 .github/workflows/zfs-linux-tests.yml         | 124 ---------
 .github/workflows/zfs-linux.yml               |  64 -----
 .github/workflows/zfs-qemu.yml                | 179 ++++++++++++
 .../cli_root/zfs_share/zfs_share_005_pos.ksh  |   1 +
 .../redacted_send/redacted_deleted.ksh        |   2 +
 16 files changed, 1133 insertions(+), 391 deletions(-)
 create mode 100644 .github/workflows/scripts/README.md
 create mode 100755 .github/workflows/scripts/qemu-1-setup.sh
 create mode 100755 .github/workflows/scripts/qemu-2-start.sh
 create mode 100755 .github/workflows/scripts/qemu-3-deps.sh
 create mode 100755 .github/workflows/scripts/qemu-4-build.sh
 create mode 100755 .github/workflows/scripts/qemu-5-setup.sh
 create mode 100755 .github/workflows/scripts/qemu-6-tests.sh
 create mode 100755 .github/workflows/scripts/qemu-7-reports.sh
 delete mode 100755 .github/workflows/scripts/setup-dependencies.sh
 delete mode 100755 .github/workflows/scripts/setup-functional.sh
 delete mode 100644 .github/workflows/zfs-linux-tests.yml
 delete mode 100644 .github/workflows/zfs-linux.yml
 create mode 100644 .github/workflows/zfs-qemu.yml

diff --git a/.github/workflows/scripts/README.md b/.github/workflows/scripts/README.md
new file mode 100644
index 000000000000..1f2f76499f05
--- /dev/null
+++ b/.github/workflows/scripts/README.md
@@ -0,0 +1,10 @@
+
+Workflow for each operating system:
+ - install QEMU on the github runner
+ - download current cloud image
+ - start and init that image via cloud-init
+ - install deps and poweroff system
+ - start system and build openzfs and then poweroff again
+ - clone the system and start 4 qemu workers for the testings (4x 3GB RAM)
+ - use trimable virtual disks (3x 1GB) for each testing system
+ - do the functional testings < 3h for each os
diff --git a/.github/workflows/scripts/generate-summary.sh b/.github/workflows/scripts/generate-summary.sh
index b5d89208a5d8..d5e6d91cc30b 100755
--- a/.github/workflows/scripts/generate-summary.sh
+++ b/.github/workflows/scripts/generate-summary.sh
@@ -1,97 +1,176 @@
 #!/usr/bin/env bash
 
-# for runtime reasons we split functional testings into N parts
-# - use a define to check for missing tarfiles
-FUNCTIONAL_PARTS="4"
-
-ZTS_REPORT="tests/test-runner/bin/zts-report.py"
-chmod +x $ZTS_REPORT
+######################################################################
+# generate github summary page of all the testings
+# /tr 2024-06-17
+######################################################################
 
 function output() {
-  echo -e $* >> Summary.md
-}
-
-function error() {
-  output ":bangbang: $* :bangbang:\n"
+  echo -e $* >> "out-$logfile.md"
 }
 
-# this function generates the real summary
-# - expects a logfile "log" in current directory
-function generate() {
-  # we issued some error already
-  test ! -s log && return
-
-  # for overview and zts-report
-  cat log | grep '^Test' > list
-
-  # error details
-  awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }
-    /\[SKIP\]|\[PASS\]/{ show=0; } show' log > err
-
-  # summary of errors
-  if [ -s err ]; then
-    output "<pre>"
-    $ZTS_REPORT --no-maybes ./list >> Summary.md
-    output "</pre>"
-
-    # generate seperate error logfile
-    ERRLOGS=$((ERRLOGS+1))
-    errfile="err-$ERRLOGS.md"
-    echo -e "\n## $headline (debugging)\n" >> $errfile
-    echo "<details><summary>Error Listing - with dmesg and dbgmsg</summary><pre>" >> $errfile
-    dd if=err bs=999k count=1 >> $errfile
-    echo "</pre></details>" >> $errfile
-  else
-    output "All tests passed :thumbsup:"
+function outfile() {
+  if [ -f $1 ]; then
+    CUR=`stat --printf="%s" "out-$logfile.md"`
+    ADD=`stat --printf="%s" "$1"`
+    X=$((CUR+ADD))
+    if [ $X -lt $((1024*1023)) ]; then
+      cat "$1" >> "out-$logfile.md"
+    else
+      logfile=$((logfile+1))
+      cat "$1" >> "out-$logfile.md"
+    fi
   fi
-
-  output "<details><summary>Full Listing</summary><pre>"
-  cat list >> Summary.md
-  output "</pre></details>"
-
-  # remove tmp files
-  rm -f err list log
 }
 
-# check tarfiles and untar
-function check_tarfile() {
-  if [ -f "$1" ]; then
-    tar xf "$1" || error "Tarfile $1 returns some error"
-  else
-    error "Tarfile $1 not found"
-  fi
+function showfile() {
+  filename="$1"
+  headline="$2"
+  echo "<details><summary>$headline</summary><pre>" > tmp
+  cat $filename >> tmp
+  echo "</pre></details>" >> tmp
+  outfile tmp
+  rm -f tmp
 }
 
-# check logfile and concatenate test results
-function check_logfile() {
-  if [ -f "$1" ]; then
-    cat "$1" >> log
-  else
-    error "Logfile $1 not found"
-  fi
+function send2github() {
+  test -f "$1" && dd if="$1" bs=1023k count=1 >> $GITHUB_STEP_SUMMARY
 }
 
-# sanity
-function summarize_s() {
-  headline="$1"
+# generate summary of one test
+function generate() {
+  VMs=3
+  ####################################################################
+  # osname.txt                       -> used for headline
+  # disk-before.txt                  -> used together with uname
+  # disk-afterwards.txt              -> used together with uname
+  # vm{1,2,3}log.txt (colored, used when current/log isn't there)
+  ####################################################################
+  # vm{1,2,3}/uname.txt              -> used once
+  # vm{1,2,3}/build-stderr.txt       -> used once
+  # vm{1,2,3}/dmesg-prerun.txt       -> used once
+  # vm{1,2,3}/dmesg-module-load.txt  -> used once
+  # vm{1,2,3}/console.txt            -> all 3 used
+  ####################################################################
+  # vm{1,2,3}/current/log     -> if not there, kernel panic loading
+  # vm{1,2,3}/current/results -> if not there, kernel panic testings
+  # vm{1,2,3}/exitcode.txt
+  ####################################################################
+
+  # headline of this summary
   output "\n## $headline\n"
-  rm -rf testfiles
-  check_tarfile "$2/sanity.tar"
-  check_logfile "testfiles/log"
-  generate
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/uname.txt ]; then
+      output "<pre>"
+      outfile vm$i/uname.txt
+      output "\nVM disk usage before:"
+      outfile disk-afterwards.txt
+      output "\nand afterwards:"
+      outfile disk-before.txt
+      output "</pre>"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/build-stderr.txt ]; then
+      showfile "vm$i/build-stderr.txt" "Module build (stderr output)"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/dmesg-prerun.txt ]; then
+      showfile "vm$i/dmesg-prerun.txt" "Dmesg output - before tests"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    if [ -s vm$i/dmesg-module-load.txt ]; then
+      showfile "vm$i/dmesg-module-load.txt" "Dmesg output - module loading"
+      break
+    fi
+  done
+
+  for i in `seq 1 $VMs`; do
+    log="vm$i/current/log"
+    if [ ! -f $log ]; then
+      output ":exclamation: Logfile of vm$i tests is missing :exclamation:"
+
+      # some out may be generated
+      if [ -s vm${i}log.txt ]; then
+        cat vm${i}log.txt | \
+          sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log"
+        showfile "vm${i}log" "Generated tests output of vm$i"
+      fi
+
+      # output the console contents and continue with next vm
+      if [ -s "vm$i/console.txt" ]; then
+        cat "vm$i/console.txt" | \
+          sed -r "s/\x1B\[([0-9]{1,2}(;[0-9]{1,2})?)?[m|K]//g" > "vm${i}log"
+        showfile "vm${i}log" "Serial console output of vm$i"
+      fi
+
+      rm -f "vm${i}log"
+      continue
+    fi
+
+    cat $log | grep '^Test[: ]' > tests.txt
+    results="vm$i/current/results"
+    if [ ! -s "$results" ]; then
+      output ":exclamation: Results file of vm$i tests is missing :exclamation:"
+      # generate results file from log
+      ./zts-report.py --no-maybes ./tests.txt > $results
+      # Running Time:	01:30:09
+      # Running Time:	not finished!!
+      echo -e "\nRunning Time:\tKernel panic!" >> $results
+    fi
+    cat $results | awk '/Results Summary/ { show=1; print; next; } show' > summary.txt
+    runtime=`cat $results | grep '^Running Time:' | cut -f2`
+
+    awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; } \
+      /\[SKIP\]|\[PASS\]/{ show=0; } show' $log > debug.txt
+
+    output "\n### Tests on vm$i ($runtime)\n\n"
+
+    if [ -s summary.txt ]; then
+      showfile "summary.txt" "Summary of all tests"
+    fi
+
+    if [ -s "vm$i/console.txt" ]; then
+      showfile "vm$i/console.txt" "Serial console output"
+    fi
+
+    if [ -s tests.txt ]; then
+      showfile "tests.txt" "List of all tests"
+    fi
+
+    MAX="300"
+    if [ -s debug.txt ]; then
+      S=`stat --printf="%s" "debug.txt"`
+      if [ $S -gt $((1024*$MAX)) ]; then
+        dd if=debug.txt of=debug.txt2 count=$MAX bs=1024 2>/dev/null
+        mv -f debug.txt2 debug.txt
+        echo "..." >> debug.txt
+        echo "!!! THIS FILE IS BIGGER !!!" >> debug.txt
+        echo "Please download the zip archiv for full content!" >> debug.txt
+      fi
+      showfile "debug.txt" "Debug list for failed tests (vm$i, $runtime)"
+    fi
+  done
 }
 
-# functional
-function summarize_f() {
-  headline="$1"
-  output "\n## $headline\n"
-  rm -rf testfiles
-  for i in $(seq 1 $FUNCTIONAL_PARTS); do
-    tarfile="$2-part$i/part$i.tar"
-    check_tarfile "$tarfile"
-    check_logfile "testfiles/log"
+# functional tests via qemu
+function summarize() {
+  for tarfile in Logs-functional-*/qemu-*.tar; do
+    rm -rf vm* *.txt
+    tar xf "$tarfile"
+    osname=`cat osname.txt`
+    headline="Functional Tests: $osname"
+    generate
   done
-  generate
 }
 
 # https://docs.github.com/en/enterprise-server@3.6/actions/using-workflows/workflow-commands-for-github-actions#step-isolation-and-limits
@@ -99,21 +178,18 @@ function summarize_f() {
 # [ ] can not show all error findings here
 # [x] split files into smaller ones and create additional steps
 
-ERRLOGS=0
-if [ ! -f Summary/Summary.md ]; then
-  # first call, we do the default summary (~500k)
-  echo -n > Summary.md
-  summarize_s "Sanity Tests Ubuntu 20.04" Logs-20.04-sanity
-  summarize_s "Sanity Tests Ubuntu 22.04" Logs-22.04-sanity
-  summarize_f "Functional Tests Ubuntu 20.04" Logs-20.04-functional
-  summarize_f "Functional Tests Ubuntu 22.04" Logs-22.04-functional
-
-  cat Summary.md >> $GITHUB_STEP_SUMMARY
-  mkdir -p Summary
-  mv *.md Summary
+# first call, generate all summaries
+if [ ! -f out-0.md ]; then
+  # create ./zts-report.py for generate()
+  TEMPLATE="tests/test-runner/bin/zts-report.py.in"
+  cat $TEMPLATE| sed -e 's|@PYTHON_SHEBANG@|python3|' > ./zts-report.py
+  chmod +x ./zts-report.py
+
+  logfile="0"
+  summarize
+  send2github out-0.md
 else
-  # here we get, when errors where returned in first call
-  test -f Summary/err-$1.md && cat Summary/err-$1.md >> $GITHUB_STEP_SUMMARY
+  send2github out-$1.md
 fi
 
 exit 0
diff --git a/.github/workflows/scripts/qemu-1-setup.sh b/.github/workflows/scripts/qemu-1-setup.sh
new file mode 100755
index 000000000000..08c9b37e2efd
--- /dev/null
+++ b/.github/workflows/scripts/qemu-1-setup.sh
@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 1) setup the action runner to start some qemu instance
+######################################################################
+
+set -eu
+
+# docker isn't needed, free some memory
+sudo systemd-run --wait docker system prune --force --all --volumes
+sudo systemctl stop docker.socket
+sudo apt-get remove docker-ce-cli docker-ce podman
+
+# remove unneeded things
+sudo apt-get remove google-chrome-stable snapd
+
+# install needed packages
+sudo apt-get update
+sudo apt-get install axel cloud-image-utils daemonize guestfs-tools \
+  virt-manager linux-modules-extra-`uname -r`
+
+# remove unused software
+df -h /
+sudo systemd-run --wait rm -rf \
+  /opt/* \
+  /usr/local/* \
+  /usr/share/az* \
+  /usr/share/dotnet \
+  /usr/share/gradle* \
+  /usr/share/miniconda \
+  /usr/share/swift \
+  /var/lib/gems \
+  /var/lib/mysql \
+  /var/lib/snapd
+
+# disk usage afterwards
+sudo df -h /
+sudo df -h /mnt
+sudo fstrim -a
+
+# generate ssh keys
+ssh-keygen -t ed25519 -f ~/.ssh/id_ed25519 -q -N ""
diff --git a/.github/workflows/scripts/qemu-2-start.sh b/.github/workflows/scripts/qemu-2-start.sh
new file mode 100755
index 000000000000..cf0198e013eb
--- /dev/null
+++ b/.github/workflows/scripts/qemu-2-start.sh
@@ -0,0 +1,160 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 2) start qemu with some operating system, init via cloud-init
+######################################################################
+
+set -eu
+
+# valid ostypes: virt-install --os-variant list
+OS="$1"
+OSv=$OS
+
+# compressed with .zst extension
+URLzs=""
+
+case "$OS" in
+  almalinux8)
+    OSNAME="AlmaLinux 8"
+    URL="https://repo.almalinux.org/almalinux/8/cloud/x86_64/images/AlmaLinux-8-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  almalinux9)
+    OSNAME="AlmaLinux 9"
+    URL="https://repo.almalinux.org/almalinux/9/cloud/x86_64/images/AlmaLinux-9-GenericCloud-latest.x86_64.qcow2"
+    ;;
+  archlinux)
+    OSNAME="Archlinux"
+    URL="https://geo.mirror.pkgbuild.com/images/latest/Arch-Linux-x86_64-cloudimg.qcow2"
+    ;;
+  centos-stream9)
+    OSNAME="CentOS Stream 9"
+    URL="https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2"
+    ;;
+  debian11)
+    OSNAME="Debian 11"
+    URL="https://cloud.debian.org/images/cloud/bullseye/latest/debian-11-generic-amd64.qcow2"
+    ;;
+  debian12)
+    OSNAME="Debian 12"
+    URL="https://cloud.debian.org/images/cloud/bookworm/latest/debian-12-generic-amd64.qcow2"
+    ;;
+  fedora39)
+    OSNAME="Fedora 39"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/39/Cloud/x86_64/images/Fedora-Cloud-Base-39-1.5.x86_64.qcow2"
+    ;;
+  fedora40)
+    OSNAME="Fedora 40"
+    OSv="fedora39"
+    URL="https://download.fedoraproject.org/pub/fedora/linux/releases/40/Cloud/x86_64/images/Fedora-Cloud-Base-Generic.x86_64-40-1.14.qcow2"
+    ;;
+  freebsd13)
+    OSNAME="FreeBSD 13"
+    OSv="freebsd13.0"
+    # URL="https://download.freebsd.org/ftp/snapshots/amd64"
+    # freebsd images don't have clout-init within it! :(
+    # -> workaround: provide own images
+    URLzs="https://openzfs.de/freebsd/amd64-freebsd-13.3.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd14)
+    OSNAME="FreeBSD 14"
+    OSv="freebsd14.0"
+    URLzs="https://openzfs.de/freebsd/amd64-freebsd-14.0.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  freebsd15)
+    OSNAME="FreeBSD 15"
+    OSv="freebsd14.0"
+    URLzs="https://openzfs.de/freebsd/amd64-freebsd-15.0.qcow2.zst"
+    BASH="/usr/local/bin/bash"
+    ;;
+  ubuntu22)
+    OSNAME="Ubuntu 22.04"
+    OSv="ubuntu22.04"
+    #URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
+    #URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img"
+    URL="https://mirror.citrahost.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img"
+    ;;
+  ubuntu24)
+    OSNAME="Ubuntu 24.04"
+    OSv="ubuntu24.04"
+    #URL="https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img"
+    #URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img"
+    URL="https://mirror.citrahost.com/ubuntu-cloud-images/noble/current/noble-server-cloudimg-amd64.img"
+    ;;
+  *)
+    echo "Wrong value for variable OS!"
+    exit 111
+    ;;
+esac
+
+IMG="/mnt/cloudimg.qcow2"
+DISK="/mnt/openzfs.qcow2"
+sudo chown -R $(whoami) /mnt
+
+if [ ! -z "$URLzs" ]; then
+  echo "Loading image $URLzs ..."
+  time axel -q -o "$IMG.zst" "$URLzs" || exit 111
+  zstd -q -d --rm "$IMG.zst"
+else
+  echo "Loading image $URL ..."
+  time axel -q -o "$IMG" "$URL" || exit 111
+fi
+
+# for later use
+echo "$OS" > /var/tmp/os.txt
+echo "$OSv" > /var/tmp/osvariant.txt
+echo "$OSNAME" > /var/tmp/osname.txt
+
+# we use zstd for faster IO on the testing runner
+echo "Converting image ..."
+qemu-img convert -q -f qcow2 -O qcow2 -c \
+  -o compression_type=zstd,preallocation=off $IMG $DISK || exit 111
+rm -f $IMG || exit 111
+
+echo "Resizing image to 20GiB ..."
+qemu-img resize -q $DISK 20G || exit 111
+
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: $OS
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+for i in `seq 0 3`; do
+  sudo virsh net-update default add ip-dhcp-host \
+    "<host mac='52:54:00:83:79:0$i' ip='192.168.122.1$i'/>" --live --config
+done
+
+sudo virt-install \
+  --os-variant $OSv \
+  --name "openzfs" \
+  --cpu host-passthrough \
+  --virt-type=kvm --hvm \
+  --vcpus=4,sockets=1 \
+  --memory $((1024*4)) \
+  --memballoon model=none \
+  --graphics none \
+  --network bridge=virbr0,model=virtio,mac='52:54:00:83:79:00' \
+  --cloud-init user-data=/tmp/user-data \
+  --disk $DISK,format=qcow2,bus=virtio \
+  --import --noautoconsole 2>/dev/null
diff --git a/.github/workflows/scripts/qemu-3-deps.sh b/.github/workflows/scripts/qemu-3-deps.sh
new file mode 100755
index 000000000000..ec06d32e3610
--- /dev/null
+++ b/.github/workflows/scripts/qemu-3-deps.sh
@@ -0,0 +1,174 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 3) install dependencies for compiling and loading
+######################################################################
+
+set -eu
+
+function archlinux() {
+  echo "##[group]Running pacman -Syu"
+  sudo pacman -Syu --noconfirm
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo pacman -Sy --noconfirm base-devel bc cpio dhclient dkms fakeroot \
+    fio gdb inetutils less linux linux-headers lsscsi nfs-utils parted pax \
+    perf python-packaging python-setuptools ksh samba sysstat rng-tools \
+    rsync wget
+  echo "##[endgroup]"
+}
+
+function debian() {
+  export DEBIAN_FRONTEND="noninteractive"
+
+  echo "##[group]Running apt-get update+upgrade"
+  sudo apt-get update -y
+  sudo apt-get upgrade -y
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo apt-get install -y \
+    acl alien attr autoconf bc cpio curl dbench dh-python \
+    dh-sequence-dkms dkms fakeroot fio gdb gdebi git ksh lcov \
+    isc-dhcp-client libacl1-dev libaio-dev libattr1-dev libblkid-dev \
+    libcurl4-openssl-dev libdevmapper-dev libelf-dev libffi-dev \
+    libmount-dev libpam0g-dev libselinux-dev libssl-dev libtool \
+    libtool-bin libudev-dev linux-headers-$(uname -r) lsscsi \
+    nfs-kernel-server pamtester parted python3 python3-all-dev \
+    python3-cffi python3-dev python3-distlib python3-packaging \
+    python3-setuptools python3-sphinx rng-tools rpm2cpio rsync samba \
+    sysstat uuid-dev watchdog wget xfslibs-dev zlib1g-dev
+  echo "##[endgroup]"
+}
+
+function freebsd() {
+  export ASSUME_ALWAYS_YES="YES"
+
+  echo "##[group]Install Development Tools"
+  sudo pkg install -y autoconf automake autotools base64 fio gdb git \
+    gmake gsed python python3 gettext gettext-runtime checkbashisms lcov \
+    libtool lscpu ksh93 pamtester pamtester rsync
+  sudo pkg install -xy \
+    '^samba4[[:digit:]]+$' \
+    '^py3[[:digit:]]+-cffi$' \
+    '^py3[[:digit:]]+-sysctl$' \
+    '^py3[[:digit:]]+-packaging$'
+  echo "##[endgroup]"
+}
+
+# common packages for: almalinux, centos, redhat
+function rhel() {
+  echo "##[group]Running dnf update"
+  sudo dnf update -y
+  echo "##[endgroup]"
+
+  echo "##[group]Install Development Tools"
+  sudo dnf group install -y "Development Tools"
+  sudo dnf install -y \
+    acl attr bc bzip2 curl dbench dkms elfutils-libelf-devel fio gdb git \
+    kernel-rpm-macros ksh libacl-devel libaio-devel libargon2-devel \
+    libattr-devel libblkid-devel libcurl-devel libffi-devel ncompress \
+    libselinux-devel libtirpc-devel libtool libudev-devel libuuid-devel \
+    lsscsi mdadm nfs-utils openssl-devel pam-devel pamtester parted perf \
+    python3 python3-cffi python3-devel python3-packaging kernel-devel \
+    python3-setuptools rng-tools rpcgen rpm-build rsync samba sysstat \
+    systemd watchdog wget xfsprogs-devel zlib-devel
+  echo "##[endgroup]"
+}
+
+# Install dependencies
+case "$1" in
+  almalinux8)
+    echo "##[group]Enable epel and powertools repositories"
+    sudo dnf config-manager -y --set-enabled powertools
+    sudo dnf install -y epel-release
+    echo "##[endgroup]"
+    rhel
+    echo "##[group]Install kernel-abi-whitelists"
+    sudo dnf install -y kernel-abi-whitelists
+    echo "##[endgroup]"
+    ;;
+  almalinux9|centos-stream9)
+    echo "##[group]Enable epel and crb repositories"
+    sudo dnf config-manager -y --set-enabled crb
+    sudo dnf install -y epel-release
+    echo "##[endgroup]"
+    rhel
+    echo "##[group]Install kernel-abi-stablelists"
+    sudo dnf install -y kernel-abi-stablelists
+    echo "##[endgroup]"
+    ;;
+  archlinux)
+    archlinux
+    ;;
+  debian*)
+    debian
+    echo "##[group]Install linux-perf"
+    sudo apt-get install -yq linux-perf
+    echo "##[endgroup]"
+    ;;
+  fedora*)
+    rhel
+    ;;
+  freebsd*)
+    freebsd
+    ;;
+  ubuntu*)
+    debian
+    echo "##[group]Install linux-tools-common"
+    sudo apt-get install -yq linux-tools-common libtirpc-dev
+    echo "##[endgroup]"
+    ;;
+esac
+
+# Start services
+echo "##[group]Enable services"
+case "$1" in
+  freebsd*)
+    echo "fdescfs /dev/fd fdescfs rw 0 0" | sudo -E tee -a /etc/fstab
+    sudo -E mount /dev/fd
+    sudo -E touch /etc/zfs/exports
+    sudo -E sysrc mountd_flags="/etc/zfs/exports"
+    sudo -E service nfsd enable
+    echo '[global]' | sudo -E tee /usr/local/etc/smb4.conf >/dev/null
+    sudo -E service samba_server enable
+    ;;
+  debian*|ubuntu*)
+    sudo -E systemctl enable nfs-kernel-server
+    sudo -E systemctl enable smbd
+    ;;
+  *)
+    # All other linux distros
+    sudo -E systemctl enable nfs-server
+    sudo -E systemctl enable smb
+    ;;
+esac
+echo "##[endgroup]"
+
+# Enable serial console and remove 'quiet' from linux kernel cmdline
+case "$1" in
+  freebsd*)
+    true
+    ;;
+  *)
+    echo "##[group]Enable serial output"
+    sudo sed -i 's/GRUB_CMDLINE_LINUX_DEFAULT="/GRUB_CMDLINE_LINUX_DEFAULT="console=ttyS0,115200n8 random.trust_cpu=on/g; s/quiet //g' /etc/default/grub || true
+    for i in /boot/grub/grub.cfg /etc/grub2.cfg /etc/grub2-efi.cfg /boot/grub2/grub.cfg ; do
+      test -e $i || continue
+      echo sudo grub-mkconfig -o $i
+      sudo grub-mkconfig -o $i
+    done
+    echo "##[endgroup]"
+    ;;
+esac
+
+# ssh config
+mkdir -p $HOME/.ssh
+echo "StrictHostKeyChecking no" >> $HOME/.ssh/config
+echo "ConnectTimeout 1" >> $HOME/.ssh/config
+
+# poweroff the machine
+# - use it for building openzfs when restarted
+sleep 2 && sudo poweroff &
+exit 0
diff --git a/.github/workflows/scripts/qemu-4-build.sh b/.github/workflows/scripts/qemu-4-build.sh
new file mode 100755
index 000000000000..4f2b9d2d7335
--- /dev/null
+++ b/.github/workflows/scripts/qemu-4-build.sh
@@ -0,0 +1,148 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 4) configure and build openzfs modules
+######################################################################
+
+set -eu
+
+# FD 3 used for stdout
+exec 3>&1
+
+function run() {
+  LOG="/var/tmp/build-stderr.txt"
+  echo "**************************************************" >> $LOG
+  echo "`date` ($*)"
+  echo "**************************************************" >> $LOG
+  $@ 2>&1 1>&3 | tee -a $LOG
+}
+
+function freebsd() {
+  export MAKE="gmake"
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr/local \
+    --with-libintl-prefix=/usr/local \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run gmake -j`sysctl -n hw.ncpu`
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo gmake install
+  echo "##[endgroup]"
+}
+
+function linux() {
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make -j$(nproc)
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo make install
+  echo "##[endgroup]"
+}
+
+function rpm_build_and_install() {
+  EXTRA_CONFIG="${1:-}"
+  echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure --enable-debug --enable-debuginfo $EXTRA_CONFIG
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make pkg-kmod pkg-utils
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  run sudo yum -y --skip-broken localinstall $(ls *.rpm | grep -v src.rpm)
+  echo "##[endgroup]"
+
+}
+
+function deb_build_and_install() {
+echo "##[group]Autogen.sh"
+  run ./autogen.sh
+  echo "##[endgroup]"
+
+  echo "##[group]Configure"
+  run ./configure \
+    --prefix=/usr \
+    --enable-pyzfs \
+    --enable-debug \
+    --enable-debuginfo
+  echo "##[endgroup]"
+
+  echo "##[group]Build"
+  run make native-deb-kmod native-deb-utils
+  echo "##[endgroup]"
+
+  echo "##[group]Install"
+  # Do kmod install.  Note that when you build the native debs, the
+  # packages themselves are placed in parent directory '../' rather than
+  # in the source directory like the rpms are.
+  run sudo apt-get -y install `find ../ | grep -E '\.deb$' | grep -Ev 'dkms|dracut'`
+  echo "##[endgroup]"
+}
+
+# Debug: show kernel cmdline
+if [ -e /proc/cmdline ] ; then
+  cat /proc/cmdline || true
+fi
+
+cd $HOME/zfs
+export PATH="$PATH:/sbin:/usr/sbin:/usr/local/sbin"
+
+# remove old configuration files
+sudo rm -rf /var/lib/cloud \
+/etc/systemd/network/10-cloud-init-eth0.network \
+/etc/network/interfaces.d/50-cloud-init.cfg \
+/etc/netplan/50-cloud-init.yaml
+
+# build
+case "$1" in
+  freebsd*)
+    freebsd
+    ;;
+  alma*|centos*)
+    rpm_build_and_install "--with-spec=redhat"
+    ;;
+  fedora*)
+    rpm_build_and_install
+    ;;
+  debian*|ubuntu*)
+    deb_build_and_install
+    ;;
+  *)
+    linux
+    ;;
+esac
+
+# poweroff the machine
+# - use it for the testings when restarted
+sync
+sleep 2 && sudo poweroff &
+exit 0
diff --git a/.github/workflows/scripts/qemu-5-setup.sh b/.github/workflows/scripts/qemu-5-setup.sh
new file mode 100755
index 000000000000..22be2555e7a6
--- /dev/null
+++ b/.github/workflows/scripts/qemu-5-setup.sh
@@ -0,0 +1,85 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 5) start test machines and load openzfs module
+######################################################################
+
+set -eu
+
+# machine not needed anymore
+while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do sleep 1; done
+sudo virsh undefine openzfs
+
+###############################################################
+# 1) 4GB RAM for host
+# 2) 3x 4GB RAM for qemu
+###############################################################
+PUBKEY=`cat ~/.ssh/id_ed25519.pub`
+OSv=`cat /var/tmp/osvariant.txt`
+OS=`cat /var/tmp/os.txt`
+for i in `seq 1 3`; do
+
+  if [ $i = 1 ]; then
+    # 8 vdisks on /mnt
+    BASE="/mnt"
+  else
+    # 4 vdisks on /
+    BASE="/mnt2"
+  fi
+
+  OPTS="-q -f qcow2 -o compression_type=zstd,preallocation=off"
+  SIZE="2"
+
+  echo "Generating vm$i with 3x${SIZE}GiB disks."
+  sudo mkdir -p $BASE
+  sudo qemu-img create $OPTS -b /mnt/openzfs.qcow2 -F qcow2 "$BASE/vm$i.qcow2"
+  sudo qemu-img create $OPTS "$BASE/vm$i-1.qcow2" ${SIZE}G
+  sudo qemu-img create $OPTS "$BASE/vm$i-2.qcow2" ${SIZE}G
+  sudo qemu-img create $OPTS "$BASE/vm$i-3.qcow2" ${SIZE}G
+
+  cat <<EOF > /tmp/user-data
+#cloud-config
+
+fqdn: vm$i
+
+# user:zfs password:1
+users:
+- name: root
+  shell: $BASH
+- name: zfs
+  sudo: ALL=(ALL) NOPASSWD:ALL
+  shell: $BASH
+  lock-passwd: false
+  passwd: \$1\$EjKAQetN\$O7Tw/rZOHaeBP1AiCliUg/
+  ssh_authorized_keys:
+    - $PUBKEY
+
+growpart:
+  mode: auto
+  devices: ['/']
+  ignore_growroot_disabled: false
+EOF
+
+  sudo virt-install \
+    --os-variant $OSv \
+    --name "vm$i" \
+    --cpu host-passthrough \
+    --virt-type=kvm --hvm \
+    --vcpus=3,sockets=1 \
+    --memory $((1024*4)) \
+    --memballoon model=none \
+    --graphics none \
+    --cloud-init user-data=/tmp/user-data \
+    --network bridge=virbr0,model=virtio,mac="52:54:00:83:79:0$i" \
+    --disk "$BASE/vm$i.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
+    --disk "$BASE/vm$i-1.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
+    --disk "$BASE/vm$i-2.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
+    --disk "$BASE/vm$i-3.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
+    --import --noautoconsole >/dev/null
+done
+
+# check if the machines are okay
+echo "Waiting for vm's to come up..."
+while true; do ssh 2>/dev/null zfs@192.168.122.11 "uname -a" && break; done
+while true; do ssh 2>/dev/null zfs@192.168.122.12 "uname -a" && break; done
+while true; do ssh 2>/dev/null zfs@192.168.122.13 "uname -a" && break; done
diff --git a/.github/workflows/scripts/qemu-6-tests.sh b/.github/workflows/scripts/qemu-6-tests.sh
new file mode 100755
index 000000000000..2840c58aeac1
--- /dev/null
+++ b/.github/workflows/scripts/qemu-6-tests.sh
@@ -0,0 +1,127 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 6) load openzfs module and run the tests
+######################################################################
+
+set -o pipefail
+
+if [ -z "$1" ]; then
+  # called directly on the runner
+  P="/var/tmp"
+
+  cd $P
+  OS=`cat os.txt`
+  IP1="192.168.122.11"
+  IP2="192.168.122.12"
+  IP3="192.168.122.13"
+
+  df -h / /mnt > /var/tmp/disk-before.txt
+
+  # start as daemon and log stdout
+  SSH=`which ssh`
+  CMD='$HOME/zfs/.github/workflows/scripts/qemu-6-tests.sh'
+  daemonize -c $P -p vm1.pid -o vm1log.txt -- \
+    $SSH zfs@$IP1 $CMD $OS part1
+  daemonize -c $P -p vm2.pid -o vm2log.txt -- \
+    $SSH zfs@$IP2 $CMD $OS part2
+  daemonize -c $P -p vm3.pid -o vm3log.txt -- \
+    $SSH zfs@$IP3 $CMD $OS part3
+
+  # give us the output of stdout + stderr - with prefix ;)
+  tail -fq vm1log.txt | sed -e "s/^/vm1: /g" &
+  tail -fq vm2log.txt | sed -e "s/^/vm2: /g" &
+  tail -fq vm3log.txt | sed -e "s/^/vm3: /g" &
+
+  # wait for all vm's to finnish
+  tail --pid=`cat vm1.pid` -f /dev/null
+  tail --pid=`cat vm2.pid` -f /dev/null
+  tail --pid=`cat vm3.pid` -f /dev/null
+
+  # kill the tail/sed combo
+  killall tail
+  df -h / /mnt > /var/tmp/disk-afterwards.txt
+  exit 0
+fi
+
+function freebsd() {
+  # when freebsd zfs is loaded, unload this one
+  kldstat -n zfs 2>/dev/null && sudo kldunload zfs
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+  sudo -E ./scripts/zfs.sh
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
+  sudo kldstat -n openzfs
+}
+
+function linux() {
+  sudo dmesg -c > /var/tmp/dmesg-prerun.txt
+  sudo -E modprobe zfs
+  sudo dmesg -c > /var/tmp/dmesg-module-load.txt
+}
+
+function gettests() {
+  TF="$TDIR/zfs-tests/tests/functional"
+  echo -n "-T "
+  case "$1" in
+    part1)
+      # ~1h 40m (archlinux)
+      echo "cli_root"
+      ;;
+    part2)
+      # ~2h 5m (archlinux)
+      ls $TF|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ','
+      ;;
+    part3)
+      # ~2h
+      ls $TF|grep '^[n-z]'|xargs|tr -s ' ' ','
+      ;;
+  esac
+}
+
+function gettestsD() {
+  TF="$TDIR/zfs-tests/tests/functional"
+  echo -n "-T "
+  case "$1" in
+    part1)
+      echo "checksum"
+      ;;
+    part2)
+      echo "casenorm,trim"
+      ;;
+    part3)
+      echo "pool_checkpoint"
+      ;;
+  esac
+}
+
+# called within vm
+export PATH="$PATH:/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/sbin:/usr/local/bin"
+
+case "$1" in
+  freebsd*)
+    TDIR="/usr/local/share/zfs"
+    freebsd
+    OPTS=`gettests $2`
+    if [ -e /dev/vtbd1 ] && [ -e /dev/vtbd2 ] && [ -e /dev/vtbd3 ] ; then
+      DISKS="/dev/vtbd1 /dev/vtbd2 /dev/vtbd3"
+      export DISKS
+    fi
+    ;;
+  *)
+    TDIR="/usr/share/zfs"
+    OPTS=`gettests $2`
+    linux
+    if [ -e /dev/vdb ] && [ -e /dev/vdc ] && [ -e /dev/vdd ] ; then
+      DISKS="/dev/vdb /dev/vdc /dev/vdd"
+      export DISKS
+    fi
+    ;;
+esac
+
+# this part runs inside qemu, finally: run tests
+uname -a > /var/tmp/uname.txt
+cd $HOME/zfs
+$TDIR/zfs-tests.sh -vKR -s 3G $OPTS | scripts/zfs-tests-color.sh
+RV=$?
+echo $RV > /var/tmp/exitcode.txt
+exit $RV
diff --git a/.github/workflows/scripts/qemu-7-reports.sh b/.github/workflows/scripts/qemu-7-reports.sh
new file mode 100755
index 000000000000..924b783d937d
--- /dev/null
+++ b/.github/workflows/scripts/qemu-7-reports.sh
@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+
+######################################################################
+# 7) output the results of the previous stage in an ordered way
+######################################################################
+
+set -o pipefail
+cd /var/tmp
+
+echo "VM disk usage before:"
+cat disk-before.txt
+echo "and afterwards:"
+cat disk-afterwards.txt
+
+exitcode=0
+for i in `seq 1 3`; do
+  f="exitcode.vm$i"
+  scp 2>/dev/null zfs@192.168.122.1$i:/var/tmp/exitcode.txt $f
+  test -f $f || echo 2 > $f
+
+  rv=`cat $f`
+  if [ $rv != 0 ]; then
+    msg="FAIL -> rv=$rv"
+    exitcode=$rv
+  else
+    msg="PASS"
+  fi
+
+  echo "##[group]Summary vm$i [$msg]"
+  cat "vm${i}log.txt" | grep -v 'Test[ :]'
+  echo "##[endgroup]"
+
+  echo "##[group]Results vm$i [$msg]"
+  cat "vm${i}log.txt"
+  echo "##[endgroup]"
+done
+
+exit $exitcode
diff --git a/.github/workflows/scripts/setup-dependencies.sh b/.github/workflows/scripts/setup-dependencies.sh
deleted file mode 100755
index b40f9290f914..000000000000
--- a/.github/workflows/scripts/setup-dependencies.sh
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-function prerun() {
-  echo "::group::Install build dependencies"
-  # remove snap things, update+upgrade will be faster then
-  for x in lxd core20 snapd; do sudo snap remove $x; done
-  sudo apt-get purge snapd google-chrome-stable firefox
-  # https://github.com/orgs/community/discussions/47863
-  sudo apt-get remove grub-efi-amd64-bin grub-efi-amd64-signed shim-signed --allow-remove-essential
-  sudo apt-get update
-  sudo apt upgrade
-  sudo xargs --arg-file=.github/workflows/build-dependencies.txt apt-get install -qq
-  sudo apt-get clean
-  sudo dmesg -c > /var/tmp/dmesg-prerun
-  echo "::endgroup::"
-}
-
-function mod_build() {
-  echo "::group::Generate debian packages"
-  ./autogen.sh
-  ./configure --enable-debug --enable-debuginfo --enable-asan --enable-ubsan
-  make --no-print-directory --silent native-deb-utils native-deb-kmod
-  mv ../*.deb .
-  rm ./openzfs-zfs-dracut*.deb ./openzfs-zfs-dkms*.deb
-  echo "$ImageOS-$ImageVersion" > tests/ImageOS.txt
-  echo "::endgroup::"
-}
-
-function mod_install() {
-  # install the pre-built module only on the same runner image
-  MOD=`cat tests/ImageOS.txt`
-  if [ "$MOD" != "$ImageOS-$ImageVersion" ]; then
-    rm -f *.deb
-    mod_build
-  fi
-
-  echo "::group::Install and load modules"
-  # don't use kernel-shipped zfs modules
-  sudo sed -i.bak 's/updates/extra updates/' /etc/depmod.d/ubuntu.conf
-  sudo apt-get install --fix-missing ./*.deb
-
-  # Native Debian packages enable and start the services
-  # Stop zfs-zed daemon, as it may interfere with some ZTS test cases
-  sudo systemctl stop zfs-zed
-  sudo depmod -a
-  sudo modprobe zfs
-  sudo dmesg
-  sudo dmesg -c > /var/tmp/dmesg-module-load
-  echo "::endgroup::"
-
-  echo "::group::Report CPU information"
-  lscpu
-  cat /proc/spl/kstat/zfs/chksum_bench
-  echo "::endgroup::"
-
-  echo "::group::Optimize storage for ZFS testings"
-  # remove swap and umount fast storage
-  # 89GiB -> rootfs + bootfs with ~80MB/s -> don't care
-  # 64GiB -> /mnt with 420MB/s -> new testing ssd
-  sudo swapoff -a
-
-  # this one is fast and mounted @ /mnt
-  # -> we reformat with ext4 + move it to /var/tmp
-  DEV="/dev/disk/azure/resource-part1"
-  sudo umount /mnt
-  sudo mkfs.ext4 -O ^has_journal -F $DEV
-  sudo mount -o noatime,barrier=0 $DEV /var/tmp
-  sudo chmod 1777 /var/tmp
-
-  # disk usage afterwards
-  sudo df -h /
-  sudo df -h /var/tmp
-  sudo fstrim -a
-  echo "::endgroup::"
-}
-
-case "$1" in
-  build)
-    prerun
-    mod_build
-    ;;
-  tests)
-    prerun
-    mod_install
-    ;;
-esac
diff --git a/.github/workflows/scripts/setup-functional.sh b/.github/workflows/scripts/setup-functional.sh
deleted file mode 100755
index 08c4d872abdf..000000000000
--- a/.github/workflows/scripts/setup-functional.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-TDIR="/usr/share/zfs/zfs-tests/tests/functional"
-echo -n "TODO="
-case "$1" in
-  part1)
-    # ~1h 20m
-    echo "cli_root"
-    ;;
-  part2)
-    # ~1h
-    ls $TDIR|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ','
-    ;;
-  part3)
-    # ~1h
-    ls $TDIR|grep '^[n-qs-z]'|xargs|tr -s ' ' ','
-    ;;
-  part4)
-    # ~1h
-    ls $TDIR|grep '^r'|xargs|tr -s ' ' ','
-    ;;
-esac
diff --git a/.github/workflows/zfs-linux-tests.yml b/.github/workflows/zfs-linux-tests.yml
deleted file mode 100644
index 753f3cd0214e..000000000000
--- a/.github/workflows/zfs-linux-tests.yml
+++ /dev/null
@@ -1,124 +0,0 @@
-name: zfs-linux-tests
-
-on:
-  workflow_call:
-    inputs:
-      os:
-        description: 'The ubuntu version: 20.02 or 22.04'
-        required: true
-        type: string
-
-jobs:
-
-  zloop:
-    runs-on: ubuntu-${{ inputs.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Tests
-      timeout-minutes: 30
-      run: |
-        sudo mkdir -p /var/tmp/zloop
-        # run for 10 minutes or at most 2 iterations for a maximum runner
-        #   time of 20 minutes.
-        sudo /usr/share/zfs/zloop.sh -t 600 -I 2 -l -m1 -- -T 120 -P 60
-    - name: Prepare artifacts
-      if: failure()
-      run: |
-        sudo chmod +r -R /var/tmp/zloop/
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: Zpool-logs-${{ inputs.os }}
-        path: |
-          /var/tmp/zloop/*/
-          !/var/tmp/zloop/*/vdev/
-        retention-days: 14
-        if-no-files-found: ignore
-    - uses: actions/upload-artifact@v4
-      if: failure()
-      with:
-        name: Zpool-files-${{ inputs.os }}
-        path: |
-          /var/tmp/zloop/*/vdev/
-        retention-days: 14
-        if-no-files-found: ignore
-
-  sanity:
-    runs-on: ubuntu-${{ inputs.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Tests
-      timeout-minutes: 60
-      shell: bash
-      run: |
-        set -o pipefail
-        /usr/share/zfs/zfs-tests.sh -vKR -s 3G -r sanity | scripts/zfs-tests-color.sh
-    - name: Prepare artifacts
-      if: success() || failure()
-      run: |
-        RESPATH="/var/tmp/test_results"
-        mv -f $RESPATH/current $RESPATH/testfiles
-        tar cf $RESPATH/sanity.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: Logs-${{ inputs.os }}-sanity
-        path: /var/tmp/test_results/sanity.tar
-        if-no-files-found: ignore
-
-  functional:
-    runs-on: ubuntu-${{ inputs.os }}
-    strategy:
-      fail-fast: false
-      matrix:
-        tests: [ part1, part2, part3, part4 ]
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - uses: actions/download-artifact@v4
-      with:
-        name: modules-${{ inputs.os }}
-    - name: Install modules
-      run: |
-        tar xzf modules-${{ inputs.os }}.tgz
-        .github/workflows/scripts/setup-dependencies.sh tests
-    - name: Setup tests
-      run: |
-        .github/workflows/scripts/setup-functional.sh ${{ matrix.tests }} >> $GITHUB_ENV
-    - name: Tests
-      timeout-minutes: 120
-      shell: bash
-      run: |
-        set -o pipefail
-        /usr/share/zfs/zfs-tests.sh -vKR -s 3G -T ${{ env.TODO }} | scripts/zfs-tests-color.sh
-    - name: Prepare artifacts
-      if: success() || failure()
-      run: |
-        RESPATH="/var/tmp/test_results"
-        mv -f $RESPATH/current $RESPATH/testfiles
-        tar cf $RESPATH/${{ matrix.tests }}.tar -h -C $RESPATH testfiles
-    - uses: actions/upload-artifact@v4
-      if: success() || failure()
-      with:
-        name: Logs-${{ inputs.os }}-functional-${{ matrix.tests }}
-        path: /var/tmp/test_results/${{ matrix.tests }}.tar
-        if-no-files-found: ignore
diff --git a/.github/workflows/zfs-linux.yml b/.github/workflows/zfs-linux.yml
deleted file mode 100644
index e6b705c86055..000000000000
--- a/.github/workflows/zfs-linux.yml
+++ /dev/null
@@ -1,64 +0,0 @@
-name: zfs-linux
-
-on:
-  push:
-  pull_request:
-
-jobs:
-
-  build:
-    name: Build
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [20.04, 22.04]
-    runs-on: ubuntu-${{ matrix.os }}
-    steps:
-    - uses: actions/checkout@v4
-      with:
-        ref: ${{ github.event.pull_request.head.sha }}
-    - name: Build modules
-      run: .github/workflows/scripts/setup-dependencies.sh build
-    - name: Prepare modules upload
-      run: tar czf modules-${{ matrix.os }}.tgz *.deb .github tests/test-runner tests/ImageOS.txt
-    - uses: actions/upload-artifact@v4
-      with:
-        name: modules-${{ matrix.os }}
-        path: modules-${{ matrix.os }}.tgz
-        retention-days: 14
-
-  testings:
-    name: Testing
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [20.04, 22.04]
-    needs: build
-    uses: ./.github/workflows/zfs-linux-tests.yml
-    with:
-      os: ${{ matrix.os }}
-
-  cleanup:
-    if: always()
-    name: Cleanup
-    runs-on: ubuntu-22.04
-    needs: testings
-    steps:
-    - uses: actions/download-artifact@v4
-    - name: Generating summary
-      run: |
-        tar xzf modules-22.04/modules-22.04.tgz .github tests
-        .github/workflows/scripts/generate-summary.sh
-    # up to 4 steps, each can have 1 MiB output (for debugging log files)
-    - name: Summary for errors #1
-      run: .github/workflows/scripts/generate-summary.sh 1
-    - name: Summary for errors #2
-      run: .github/workflows/scripts/generate-summary.sh 2
-    - name: Summary for errors #3
-      run: .github/workflows/scripts/generate-summary.sh 3
-    - name: Summary for errors #4
-      run: .github/workflows/scripts/generate-summary.sh 4
-    - uses: actions/upload-artifact@v4
-      with:
-        name: Summary Files
-        path: Summary/
diff --git a/.github/workflows/zfs-qemu.yml b/.github/workflows/zfs-qemu.yml
new file mode 100644
index 000000000000..52b972d13a7a
--- /dev/null
+++ b/.github/workflows/zfs-qemu.yml
@@ -0,0 +1,179 @@
+name: zfs-qemu
+
+on:
+  push:
+  pull_request:
+
+jobs:
+
+  qemu-vm:
+    name: QEMU
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24]
+    runs-on: ubuntu-24.04
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+
+    - name: Setup QEMU
+      timeout-minutes: 10
+      run: .github/workflows/scripts/qemu-1-setup.sh
+
+    - name: Start build machine
+      timeout-minutes: 5
+      run: .github/workflows/scripts/qemu-2-start.sh ${{ matrix.os }}
+
+    - name: Install dependencies
+      timeout-minutes: 20
+      run: |
+        echo "Install dependencies in QEMU machine"
+        echo "ConnectTimeout 1" >> $HOME/.ssh/config
+        echo "StrictHostKeyChecking no" >> $HOME/.ssh/config
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          ssh 2>/dev/null zfs@192.168.122.10 "uname -a" && break
+        done
+        scp .github/workflows/scripts/qemu-3-deps.sh zfs@192.168.122.10:qemu-3-deps.sh
+        PID=`pidof /usr/bin/qemu-system-x86_64`
+        ssh zfs@192.168.122.10 '$HOME/qemu-3-deps.sh' ${{ matrix.os }}
+        # wait for poweroff to succeed
+        tail --pid=$PID -f /dev/null
+        sleep 5 # avoid this: "error: Domain is already active"
+
+    - name: Build modules
+      timeout-minutes: 30
+      run: |
+        echo "Build modules in QEMU machine"
+        sudo virsh start openzfs
+        while pidof /usr/bin/qemu-system-x86_64 >/dev/null; do
+          ssh 2>/dev/null zfs@192.168.122.10 "uname -a" && break
+        done
+        rsync -ar $HOME/work/zfs/zfs zfs@192.168.122.10:./
+        ssh zfs@192.168.122.10 '$HOME/zfs/.github/workflows/scripts/qemu-4-build.sh' ${{ matrix.os }}
+
+    - name: Setup testing machines
+      timeout-minutes: 5
+      run: |
+        .github/workflows/scripts/qemu-5-setup.sh
+        # Save the VM's serial output (ttyS0) to /var/tmp/console.txt
+        # - ttyS0 on the VM corresponds to a local /dev/pty/N entry
+        # - use 'virsh ttyconsole' to lookup the /dev/pty/N entry
+        RESPATH="/var/tmp/test_results"
+        mkdir -p $RESPATH/vm{1,2,3}
+        sudo mkdir -p /var/tmp
+        read "pty1" <<< $(sudo virsh ttyconsole "vm1")
+        read "pty2" <<< $(sudo virsh ttyconsole "vm2")
+        read "pty3" <<< $(sudo virsh ttyconsole "vm3")
+        sudo nohup bash -c "cat "$pty1" > $RESPATH/vm1/console.txt" &
+        sudo nohup bash -c "cat "$pty2" > $RESPATH/vm2/console.txt" &
+        sudo nohup bash -c "cat "$pty3" > $RESPATH/vm3/console.txt" &
+        echo "Console logging for $pty1, $pty2 and $pty3 started."
+
+    - name: Run tests
+      timeout-minutes: 210
+      run: |
+        .github/workflows/scripts/qemu-6-tests.sh
+
+    - name: Test reports
+      timeout-minutes: 10
+      run: |
+        .github/workflows/scripts/qemu-7-reports.sh
+
+    - name: Prepare artifacts
+      if: success() || failure()
+      run: |
+        RESPATH="/var/tmp/test_results"
+        rsync -arL zfs@192.168.122.11:$RESPATH/current $RESPATH/vm1 || true
+        rsync -arL zfs@192.168.122.12:$RESPATH/current $RESPATH/vm2 || true
+        rsync -arL zfs@192.168.122.13:$RESPATH/current $RESPATH/vm3 || true
+        scp zfs@192.168.122.11:"/var/tmp/*.txt" $RESPATH/vm1 || true
+        scp zfs@192.168.122.12:"/var/tmp/*.txt" $RESPATH/vm2 || true
+        scp zfs@192.168.122.13:"/var/tmp/*.txt" $RESPATH/vm3 || true
+        cp -f /var/tmp/*.txt $RESPATH || true
+        ls -la /var/tmp
+        tar cf qemu-${{ matrix.os }}.tar -C $RESPATH -h . || true
+
+    - uses: actions/upload-artifact@v4
+      if: success() || failure()
+      with:
+        name: Logs-functional-${{ matrix.os }}
+        path: qemu-${{ matrix.os }}.tar
+        if-no-files-found: ignore
+
+  cleanup:
+    if: always()
+    name: Cleanup
+    runs-on: ubuntu-latest
+    needs: [ qemu-vm ]
+
+    steps:
+    - uses: actions/checkout@v4
+      with:
+        ref: ${{ github.event.pull_request.head.sha }}
+    - uses: actions/download-artifact@v4
+    - name: Generating summary
+      run: .github/workflows/scripts/generate-summary.sh
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 1
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 2
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 3
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 4
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 5
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 6
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 7
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 8
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 9
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 10
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 11
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 12
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 13
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 14
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 15
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 16
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 17
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 18
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 19
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 20
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 21
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 22
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 23
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 24
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 25
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 26
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 27
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 28
+    - name: Generating summary...
+      run: .github/workflows/scripts/generate-summary.sh 29
+    - uses: actions/upload-artifact@v4
+      with:
+        name: Summary Files
+        path: out-*
diff --git a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
index 85854085f560..b5d8a4fc32ae 100755
--- a/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
+++ b/tests/zfs-tests/tests/functional/cli_root/zfs_share/zfs_share_005_pos.ksh
@@ -67,6 +67,7 @@ log_onexit cleanup
 
 cleanup
 
+
 typeset -i i=0
 while (( i < ${#shareopts[*]} ))
 do
diff --git a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
index ec11610742fb..e7588e207b46 100755
--- a/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
+++ b/tests/zfs-tests/tests/functional/redacted_send/redacted_deleted.ksh
@@ -68,6 +68,7 @@ log_must mount_redacted -f $recvfs
 # when we re-enable redaction blkptrs.
 #
 #log_mustnot dd if=$recv_mnt/f1 of=/dev/null bs=512 count=1
+log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt"
 log_must diff $send_mnt/f2 $recv_mnt/f2
 log_must zfs rollback -R $clone@snap
 log_must zfs destroy -R $recvfs
@@ -96,6 +97,7 @@ log_must zfs destroy -R $clone2
 log_must eval "zfs send -i $sendfs#book2 --redact book3 $sendfs@snap2 >$stream"
 log_must eval "zfs recv $recvfs <$stream"
 log_must mount_redacted -f $recvfs
+log_note "send_mnt1=$send_mnt, recv_mnt1=$recv_mnt"
 log_must [ "$(ls $send_mnt)" == "$(ls $recv_mnt)" ]
 log_must zfs destroy -R $recvfs
 log_must zfs rollback -R $sendfs@snap