Skip to content

Commit

Permalink
Merge summaries
Browse files Browse the repository at this point in the history
Just testing

Requires-builders: none
Signed-off-by: Tony Hutter <hutter2@llnl.gov>
  • Loading branch information
tonyhutter committed Jun 25, 2024
1 parent e4f4605 commit 73f5c57
Show file tree
Hide file tree
Showing 7 changed files with 210 additions and 33 deletions.
127 changes: 127 additions & 0 deletions .github/workflows/scripts/merge_summary.awk
Original file line number Diff line number Diff line change
@@ -0,0 +1,127 @@
#!/bin/awk -f
#
# Merge multiple ZTS tests results summaries into a single summary. This is
# needed when you're running different parts of ZTS on different tests
# runners or VMs.
#
# Usage:
#
# ./merge_summary.awk summary1.txt [summary2.txt] [summary3.txt] ...
#
# or:
#
# cat summary*.txt | ./merge_summary.awk
#
# NOTE!!! Make sure all ANSI color is removed before running this script or
# it's not going to work correctly. To remove ANSI colors:
#
# sed -e 's/\x1b\[[0-9;]*m//g'
#
BEGIN {
i=-1
pass=0
fail=0
skip=0
state="config_lines"
cl=0
el=0
epl=0
ul=0

# Total seconds of tests runtime
total=0;
}

/Configuration/{
i++;
if (state != "config_lines") {
# new file, clear our state
state="";
}
}

# Skip empty lines
/^\s*$/{next}

# When we see "test-runner.py" stop saving config lines, and
# save test runner lines
/test-runner.py/{state="testrunner"; runner=runner$0"\n"; next}

# We need to differentiate the PASS counts from test result lines that start
# with PASS, like:
#
# PASS mv_files/setup
#
# Use state="pass_count" to differentiate
#
/Results Summary/{state="pass_count"; next}
/PASS/{ if (state=="pass_count") {pass += $2}}
/FAIL/{ if (state=="pass_count") {fail += $2}}
/SKIP/{ if (state=="pass_count") {skip += $2}}
/Running Time/{
state="";
running[i]=$3;
split($3, arr, ":")
total += arr[1] * 60 * 60;
total += arr[2] * 60;
total += arr[3]
next;
}

# Just save the log directory from the first summary since we probably don't
# care what the value is.
/Log directory/{if (i == 0) {logdir_line=$0"\n"}; next}
/Tests with results other than PASS that are expected/{state="expected_lines"; next}
/Tests with result of PASS that are unexpected/{state="unexpected_pass_lines"; next}
/Tests with results other than PASS that are unexpected/{state="unexpected_lines"; next}
{
# Save the opening configuration lines from first summary file. These
# should be relatively common to all the summaries.
if (state == "config_lines") {
config_lines[cl] = $0
cl++;
}

if (state == "expected_lines") {
expected_lines[el] = $0
el++
}

if (state == "unexpected_pass_lines") {
unexpected_pass_lines[upl] = $0
upl++
}
if (state == "unexpected_lines") {
unexpected_lines[ul] = $0
ul++
}
}

# Reproduce summary
END {
for (j in config_lines)
print config_lines[j]
print ""
print runner;
print " Results Summary"
print " PASS\t"pass
print " FAIL\t"fail
print " SKIP\t"skip
print ""
print " Running Time:\t"strftime("%T", total, 1)

percent_passed=(pass/(pass+fail+skip) * 100)
printf " Percent passed:\t%3.2f%\n", percent_passed
print logdir_line
print " Tests with results other than PASS that are expected:"
for (j in expected_lines)
print expected_lines[j]
print ""
print " Tests with result of PASS that are unexpected:"
for (j in unexpected_pass_lines)
print unexpected_pass_lines[j]
print ""
print " Tests with results other than PASS that are unexpected:"
for (j in unexpected_lines)
print unexpected_lines[j]
}
34 changes: 20 additions & 14 deletions .github/workflows/scripts/qemu-2-start.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ OS="$1"
OSv=$OS

# compressed with .zst extension
BASE="https://github.com/mcmilk/openzfs-freebsd-images/releases/download/v2024-06-20v2"
URLzs=""

case "$OS" in
Expand Down Expand Up @@ -51,30 +52,34 @@ case "$OS" in
freebsd13)
OSNAME="FreeBSD 13"
OSv="freebsd13.0"
# URL="https://download.freebsd.org/ftp/snapshots/amd64"
# freebsd images don't have clout-init within it! :(
# -> workaround: provide own images
URLzs="https://openzfs.de/freebsd/amd64-freebsd-13.3.qcow2.zst"
URLzs="$BASE/amd64-freebsd-13.3-STABLE.qcow2.zst"
BASH="/usr/local/bin/bash"
;;
freebsd14)
OSNAME="FreeBSD 14"
OSv="freebsd14.0"
URLzs="https://openzfs.de/freebsd/amd64-freebsd-14.0.qcow2.zst"
URLzs="$BASE/amd64-freebsd-14.1-STABLE.qcow2.zst"
BASH="/usr/local/bin/bash"
;;
freebsd15)
OSNAME="FreeBSD 15"
OSv="freebsd14.0"
URLzs="https://openzfs.de/freebsd/amd64-freebsd-15.0.qcow2.zst"
URLzs="$BASE/amd64-freebsd-15.0-CURRENT.qcow2.zst"
BASH="/usr/local/bin/bash"
;;
tumbleweed)
OSNAME="openSUSE Tumbleweed"
OSv="opensusetumbleweed"
MIRROR="http://opensuse-mirror-gce-us.susecloud.net"
URL="$MIRROR/tumbleweed/appliances/openSUSE-MicroOS.x86_64-OpenStack-Cloud.qcow2"
;;
ubuntu22)
OSNAME="Ubuntu 22.04"
OSv="ubuntu22.04"
#URL="https://cloud-images.ubuntu.com/jammy/current/jammy-server-cloudimg-amd64.img"
#URL="https://mirrors.cloud.tencent.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img"
URL="https://mirror.citrahost.com/ubuntu-cloud-images/jammy/current/jammy-server-cloudimg-amd64.img"
MIRROR="https://cloud-images.ubuntu.com"
MIRROR="https://mirrors.cloud.tencent.com/ubuntu-cloud-images"
MIRROR="https://mirror.citrahost.com/ubuntu-cloud-images"
URL="$MIRROR/jammy/current/jammy-server-cloudimg-amd64.img"
;;
ubuntu24)
OSNAME="Ubuntu 24.04"
Expand Down Expand Up @@ -113,8 +118,8 @@ qemu-img convert -q -f qcow2 -O qcow2 -c \
-o compression_type=zstd,preallocation=off $IMG $DISK || exit 111
rm -f $IMG || exit 111

echo "Resizing image to 20GiB ..."
qemu-img resize -q $DISK 20G || exit 111
echo "Resizing image to 50GiB ..."
qemu-img resize -q $DISK 50G || exit 111

PUBKEY=`cat ~/.ssh/id_ed25519.pub`
cat <<EOF > /tmp/user-data
Expand Down Expand Up @@ -145,6 +150,7 @@ for i in `seq 0 3`; do
"<host mac='52:54:00:83:79:0$i' ip='192.168.122.1$i'/>" --live --config
done

# We need 'model=e1000' for FreeBSD 13
sudo virt-install \
--os-variant $OSv \
--name "openzfs" \
Expand All @@ -154,7 +160,7 @@ sudo virt-install \
--memory $((1024*4)) \
--memballoon model=none \
--graphics none \
--network bridge=virbr0,model=virtio,mac='52:54:00:83:79:00' \
--network bridge=virbr0,model=e1000,mac='52:54:00:83:79:00' \
--cloud-init user-data=/tmp/user-data \
--disk $DISK,format=qcow2,bus=virtio \
--import --noautoconsole 2>/dev/null
--disk $DISK,format=qcow2,bus=e1000 \
--import --noautoconsole
12 changes: 12 additions & 0 deletions .github/workflows/scripts/qemu-3-deps.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@

set -eu

OS="$1"

function archlinux() {
echo "##[group]Running pacman -Syu"
sudo pacman -Syu --noconfirm
Expand Down Expand Up @@ -45,6 +47,16 @@ function debian() {
function freebsd() {
export ASSUME_ALWAYS_YES="YES"

# if [ "$OS" == "freebsd13" ] ; then
# echo "FreeBSD 13 bootstrap pkg"
# ping -c10 www.FreeBSD.org || true
# which wget || true
#
# sudo pkg-static bootstrap -f || true
# sudo pkg bootstrap -f || true
# sudo pkg update -f || true
# fi

echo "##[group]Install Development Tools"
sudo pkg install -y autoconf automake autotools base64 fio gdb git \
gmake gsed python python3 gettext gettext-runtime checkbashisms lcov \
Expand Down
14 changes: 8 additions & 6 deletions .github/workflows/scripts/qemu-5-setup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@ for i in `seq 1 3`; do
echo "Generating vm$i with 3x${SIZE}GiB disks."
sudo mkdir -p $BASE
sudo qemu-img create $OPTS -b /mnt/openzfs.qcow2 -F qcow2 "$BASE/vm$i.qcow2"
sudo qemu-img create $OPTS "$BASE/vm$i-1.qcow2" ${SIZE}G
sudo qemu-img create $OPTS "$BASE/vm$i-2.qcow2" ${SIZE}G
sudo qemu-img create $OPTS "$BASE/vm$i-3.qcow2" ${SIZE}G
# sudo qemu-img create $OPTS "$BASE/vm$i-1.qcow2" ${SIZE}G
# sudo qemu-img create $OPTS "$BASE/vm$i-2.qcow2" ${SIZE}G
# sudo qemu-img create $OPTS "$BASE/vm$i-3.qcow2" ${SIZE}G

cat <<EOF > /tmp/user-data
#cloud-config
Expand Down Expand Up @@ -72,10 +72,12 @@ EOF
--cloud-init user-data=/tmp/user-data \
--network bridge=virbr0,model=virtio,mac="52:54:00:83:79:0$i" \
--disk "$BASE/vm$i.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
--disk "$BASE/vm$i-1.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
--disk "$BASE/vm$i-2.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
--disk "$BASE/vm$i-3.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
--import --noautoconsole >/dev/null

# --disk "$BASE/vm$i-1.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
# --disk "$BASE/vm$i-2.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
# --disk "$BASE/vm$i-3.qcow2",format=qcow2,bus=virtio,driver.discard=unmap \
# --import --noautoconsole >/dev/null
done

# check if the machines are okay
Expand Down
19 changes: 11 additions & 8 deletions .github/workflows/scripts/qemu-6-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -66,14 +66,17 @@ function gettests() {
part1)
# ~1h 40m (archlinux)
echo "cli_root"
# echo "zpool_add,zpool_create,zpool_export"
;;
part2)
# ~2h 5m (archlinux)
ls $TF|grep '^[a-m]'|grep -v "cli_root"|xargs|tr -s ' ' ','
# echo "zfs_receive,zpool_initialize"
;;
part3)
# ~2h
ls $TF|grep '^[n-z]'|xargs|tr -s ' ' ','
# echo "zfs_unshare,zpool_destroy"
;;
esac
}
Expand Down Expand Up @@ -102,19 +105,19 @@ case "$1" in
TDIR="/usr/local/share/zfs"
freebsd
OPTS=`gettests $2`
if [ -e /dev/vtbd1 ] && [ -e /dev/vtbd2 ] && [ -e /dev/vtbd3 ] ; then
DISKS="/dev/vtbd1 /dev/vtbd2 /dev/vtbd3"
export DISKS
fi
# if [ -e /dev/vtbd1 ] && [ -e /dev/vtbd2 ] && [ -e /dev/vtbd3 ] ; then
# DISKS="/dev/vtbd1 /dev/vtbd2 /dev/vtbd3"
# export DISKS
# fi
;;
*)
TDIR="/usr/share/zfs"
OPTS=`gettests $2`
linux
if [ -e /dev/vdb ] && [ -e /dev/vdc ] && [ -e /dev/vdd ] ; then
DISKS="/dev/vdb /dev/vdc /dev/vdd"
export DISKS
fi
# if [ -e /dev/vdb ] && [ -e /dev/vdc ] && [ -e /dev/vdd ] ; then
# DISKS="/dev/vdb /dev/vdc /dev/vdd"
# export DISKS
# fi
;;
esac

Expand Down
13 changes: 13 additions & 0 deletions .github/workflows/scripts/qemu-7-reports.sh
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
######################################################################

set -o pipefail
ZFSDIR="$(pwd)"
cd /var/tmp

echo "VM disk usage before:"
Expand Down Expand Up @@ -35,4 +36,16 @@ for i in `seq 1 3`; do
echo "##[endgroup]"
done

# Merge all summaries
echo "Merging summaries1, zfsdir $ZFSDIR"
echo "current dir: $(ls -l)"
echo "homedir dir: $(ls -l ~)"

# The 'sed' line here removes ANSI color. This is needed for merge_summary.awk
# to work. We add the color back in on the final line.
cat vm*log.txt | grep -v 'Test[ :]' | \
sed -e 's/\x1b\[[0-9;]*m//g' | \
$ZFSDIR/.github/workflows/scripts/merge_summary.awk | \
$ZFSDIR/scripts/zfs-tests-color.sh

exit $exitcode
24 changes: 19 additions & 5 deletions .github/workflows/zfs-qemu.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,10 @@ jobs:
strategy:
fail-fast: false
matrix:
os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24]
# os: [almalinux8, almalinux9, archlinux, centos-stream9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24]
# os: [almalinux8, almalinux9, fedora39, fedora40, debian11, debian12, freebsd13, freebsd14, freebsd15, ubuntu22, ubuntu24]
os: [freebsd13]

runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -71,10 +74,10 @@ jobs:
sudo nohup bash -c "cat "$pty3" > $RESPATH/vm3/console.txt" &
echo "Console logging for $pty1, $pty2 and $pty3 started."
- name: Run tests
timeout-minutes: 210
run: |
.github/workflows/scripts/qemu-6-tests.sh
# - name: Run tests
# timeout-minutes: 210
# run: |
# .github/workflows/scripts/qemu-6-tests.sh

- name: Test reports
timeout-minutes: 10
Expand Down Expand Up @@ -102,6 +105,17 @@ jobs:
path: qemu-${{ matrix.os }}.tar
if-no-files-found: ignore

- name: Simple test results
if: success() || failure()
run: |
ZFSDIR="$(pwd)"
cd /var/tmp/test_results
cat vm*log.txt | grep -v 'Test[ :]' | \
sed -e 's/\x1b\[[0-9;]*m//g' | \
$ZFSDIR/.github/workflows/scripts/merge_summary.awk | \
$ZFSDIR/scripts/zfs-tests-color.sh
awk '/\[FAIL\]|\[KILLED\]/{ show=1; print; next; }; /\[SKIP\]|\[PASS\]/{ show=0; } show' ./vm*/current/log
cleanup:
if: always()
name: Cleanup
Expand Down

0 comments on commit 73f5c57

Please sign in to comment.