From 0e2fad03130871acc80699b56f5914271ed55402 Mon Sep 17 00:00:00 2001 From: Russ Cox Date: Fri, 22 Jul 2022 12:16:17 -0400 Subject: [PATCH] env/dragonfly-amd64: add scripts for building GCE VM image Now that Dragonfly runs on GCE, we can do that and retire the one very slow reverse builder we are using today. For golang/go#23060. Change-Id: I2bd8c8be6735212ba6a8023327864b79dea08cf3 Reviewed-on: https://go-review.googlesource.com/c/build/+/419081 Auto-Submit: Russ Cox Run-TryBot: Russ Cox TryBot-Result: Gopher Robot Reviewed-by: Heschi Kreinick --- env/dragonfly-amd64/.gitignore | 5 ++ env/dragonfly-amd64/Makefile | 110 +++++++++++++++++++++++++++++++++ env/dragonfly-amd64/README | 28 +++++++++ env/dragonfly-amd64/buildlet | 79 +++++++++++++++++++++++ env/dragonfly-amd64/phase1.sh | 60 ++++++++++++++++++ env/dragonfly-amd64/phase2.sh | 27 ++++++++ env/dragonfly-amd64/phase3.sh | 47 ++++++++++++++ env/freebsd-amd64/buildlet | 5 +- 8 files changed, 357 insertions(+), 4 deletions(-) create mode 100644 env/dragonfly-amd64/.gitignore create mode 100644 env/dragonfly-amd64/Makefile create mode 100644 env/dragonfly-amd64/README create mode 100644 env/dragonfly-amd64/buildlet create mode 100644 env/dragonfly-amd64/phase1.sh create mode 100644 env/dragonfly-amd64/phase2.sh create mode 100644 env/dragonfly-amd64/phase3.sh diff --git a/env/dragonfly-amd64/.gitignore b/env/dragonfly-amd64/.gitignore new file mode 100644 index 0000000000..a11d4d5ffa --- /dev/null +++ b/env/dragonfly-amd64/.gitignore @@ -0,0 +1,5 @@ +*.iso +*.iso.bz2 +disk-* +disk.* +isotmp diff --git a/env/dragonfly-amd64/Makefile b/env/dragonfly-amd64/Makefile new file mode 100644 index 0000000000..ee8217d0e5 --- /dev/null +++ b/env/dragonfly-amd64/Makefile @@ -0,0 +1,110 @@ +# See README for an overview of how to use this Makefile. + +# Database of ISO names and MD5s. Variable name suffix is short version (622 for 6.2.2, see V below) +ISO_600=dfly-x86_64-6.0.0_REL.iso +MD5_600=252fc700803dbadce1de928432373bbb + +ISO_622=dfly-x86_64-6.2.2_REL.iso +MD5_622=b391a29c8e9c1cc33f2441ff1f0d98a0 + +# Default version when invoking make. Can override on command line: make V=600 +V=622 + +ISO=$(ISO_$V) +MD5=$(MD5_$V) + +default: disk-$V.tar.gz + +clean: + rm -f *.iso *.iso.bz2 phase*.iso isotmp disk-*.phase* disk-*.tar.gz disk.raw disk.tmp + +# Download image from DragonflyBSD page. +$(ISO): + curl -o $@.bz2 https://mirror-master.dragonflybsd.org/iso-images/$@.bz2 + echo "$(MD5) $@.bz2" | md5sum -c - + bunzip2 $@.bz2 + +# If Dragonfly boots with a CD containing pfi.conf that says pfi_script=name.sh, +# then it runs name.sh at the end of the standard startup process. +# We use this to run a script inside a QEMU image. +# phase1.iso is a CD containing phase1.sh, and so on. +%.iso: %.sh + rm -rf isotmp + mkdir isotmp + cp $^ isotmp + echo pfi_script=$*.sh >>isotmp/pfi.conf + chmod +x isotmp/$*.sh + rm -f $@ + genisoimage -r -o $@ isotmp/ + rm -rf isotmp + +# Disk image creation is broken into phases so that later phases +# can be modified and rerun without repeating the earlier phases, +# which can take a long time. + +# Phase 1 sets up a basic DragonflyBSD install on the disk. +# The installer is CD0, the PFI script is CD1, and the installer is also CD2. +# CD2 is used by phase1.sh to get a clean FS image with nothing mounted on it. +disk-$V.phase1: $(ISO) phase1.iso + qemu-img create -f qcow2 $@ 16G + (sleep 1; echo 9; echo set console=comconsole; echo boot) | \ + qemu-system-x86_64 \ + -display none -serial stdio \ + -m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \ + -drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \ + -drive file=$(ISO),media=cdrom \ + -drive file=phase1.iso,media=cdrom \ + -drive file=$(ISO),media=cdrom + +# Phase 2 updates the pkg database and installing curl, git, and so on. +disk-$V.phase2: disk-$V.phase1 phase2.iso + cp $< $@ + qemu-system-x86_64 \ + -display none -serial stdio \ + -m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \ + -drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \ + -drive file=phase2.iso,media=cdrom + +# Phase 3 customizes the image to run the buildlet. +phase3.iso: buildlet +disk-$V.phase3: disk-$V.phase2 phase3.iso + cp $< $@ + qemu-system-x86_64 \ + -display none -serial stdio \ + -m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \ + -drive file=$@,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 \ + -drive file=phase3.iso,media=cdrom + +# For testing and exploration +run: disk-$V.phase3 + cp $< disk.tmp + qemu-system-x86_64 \ + -display none -serial stdio \ + -m 4G -net nic,model=virtio -net user -device virtio-scsi-pci,id=scsi0 \ + -drive file=disk.tmp,if=none,format=qcow2,cache=none,id=myscsi -device scsi-hd,drive=myscsi,bus=scsi0.0 + +# Convert QEMU image back to raw for Google Cloud. +disk-$V.tar.gz: disk-$V.phase3 + rm -f disk.raw + qemu-img convert -f qcow2 -O raw -t none -T none $< disk.raw + tar -Sczf $@ disk.raw + rm -f disk.raw + +# Upload and create prod disk image. +upload-prod: disk-$V.tar.gz + @echo "Run 'gcloud auth application-default login' if you get credential errors." + go run ../../cmd/upload --verbose --file=$< --public go-builder-data/dragonfly-amd64-$V.tar.gz + gcloud compute --project symbolic-datum-552 images create dragonfly-amd64-$V --source-uri gs://go-builder-data/dragonfly-amd64-$V.tar.gz + +# Delete prod disk image. +delete-prod: + gcloud compute --project symbolic-datum-552 images delete dragonfly-amd64-$V + +# Upload and create staging disk image. +upload-staging: disk-$V.tar.gz + go run ../../cmd/upload --verbose --file=$< --public dev-go-builder-data/dragonfly-amd64-$V.tar.gz + gcloud compute --project go-dashboard-dev images create dragonfly-amd64-$V --source-uri gs://dev-go-builder-data/dragonfly-amd64-$V.tar.gz + +# Delete staging disk image. +delete-staging: + gcloud compute --project go-dashboard-dev images delete dragonfly-amd64-$V diff --git a/env/dragonfly-amd64/README b/env/dragonfly-amd64/README new file mode 100644 index 0000000000..319c4bfb93 --- /dev/null +++ b/env/dragonfly-amd64/README @@ -0,0 +1,28 @@ +The Makefile in this directory creates a Google Compute Engine VM image to run the Go +DragonflyBSD builder, booting up to run the buildlet. + +make must be run on a Linux box with qemu and a few additional packages: + + apt-get install qemu-system-x86 qemu-utils expect genisoimage + +To override the default Dragonfly version, set V= on the make command line: + + make V=600 + +To add new Dragonfly versions, add lines to the "Database" in the Makefile +and perhaps change the V= line to update the default. + + make upload-prod + +builds and uploads the image. To replace an existing image, use + + make delete-prod upload-prod + +s/prod/staging/ to test in the staging cluster instead. + +The VM needs to be run with the GCE metadata attribute "buildlet-binary-url" set to a URL +of the DragonflyBSD buildlet (cross-compiled, typically). + + buildlet-binary-url == http://storage.googleapis.com/go-builder-data/buildlet.dragonfly-amd64 + +The buildlet rc script is lightly modified from ../freebsd-amd64. diff --git a/env/dragonfly-amd64/buildlet b/env/dragonfly-amd64/buildlet new file mode 100644 index 0000000000..d30e52453f --- /dev/null +++ b/env/dragonfly-amd64/buildlet @@ -0,0 +1,79 @@ +#!/bin/sh +# PROVIDE: buildlet +# REQUIRE: NETWORKING DAEMON +# BEFORE: LOGIN + +. /etc/rc.subr + +name=buildlet +rcvar=buildlet_enable + +procname=/buildlet +pidfile="/var/run/${name}.pid" +command=/usr/sbin/daemon +command_args="-r -fc -p ${pidfile}" +command_args="${command_args} ${procname}" +start_precmd="${name}_prestart" +stop_cmd=":" + +load_rc_config $name +: ${buildlet_enable:="NO"} + +buildlet_resetnet() +{ + warn "buildlet resetting network" + ifconfig vtnet0 down + sleep 1 + ifconfig vtnet0 up + sleep 1 + dhclient vtnet0 + sleep 1 +} + +buildlet_fixnet() +{ + # Some fraction of the time, the VM comes up unable to deliver UDP packets. + # If we detect this situation, by host -W 3 failing to look up metadata.google.internal, + # then reset the network interface (ifup/ifdown/dhclient). + # Once is almost always enough, so 10 attempts should drive the failure rate to zero. + for i in 0 1 2 3 4 5 6 7 8 9; do + if /usr/local/bin/host -W 3 metadata.google.internal; then + return 0 + fi + buildlet_resetnet + done + return 1 +} + +buildlet_prestart() +{ + local buildlet_url + + info $(netstat -rn) + info $(cat /etc/resolv.conf) + + if ! buildlet_fixnet; then + warn "cannot fix network" + poweroff + return 1 + fi + + buildlet_url=$(/usr/local/bin/curl -s -H "Metadata-Flavor: Google" \ + http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url) + + if [ "$buildlet_url" = "" ]; then + warn "cannot find buildlet url" + poweroff + return 1 + fi + + if ! /usr/local/bin/curl -o /buildlet "${buildlet_url}"; then + warn "failed to download buildlet from ${buildlet_url}" + poweroff + return 1 + fi + + chmod a+x /buildlet +} + +run_rc_command "$1" diff --git a/env/dragonfly-amd64/phase1.sh b/env/dragonfly-amd64/phase1.sh new file mode 100644 index 0000000000..c435459443 --- /dev/null +++ b/env/dragonfly-amd64/phase1.sh @@ -0,0 +1,60 @@ +#!/bin/sh + +# Phase 1 of the DragonflyBSD installation: boot from installer CD and install on empty disk. + +set -ex + +echo >&2 phase1.sh starting + +mkdir -p /root/gopherinstall +cp -a /mnt/* /root/gopherinstall +cd /root/gopherinstall + +echo >&2 install.sh running + +# Following Manual Installation section of https://www.dragonflybsd.org/docs/handbook/Installation/#index3h1 + +fdisk -IB /dev/da0 +disklabel -r -w -B /dev/da0s1 auto +disklabel da0s1 >label +echo ' +a: 1G 0 4.2BSD +d: * * HAMMER2 +' >>label +disklabel -R -r da0s1 label + +newfs /dev/da0s1a +newfs_hammer2 -L ROOT /dev/da0s1d +mount /dev/da0s1d /mnt +mkdir /mnt/boot /mnt/mnt +mount /dev/da0s1a /mnt/boot + +# mount clean file system image from second copy of CD on /mnt/mnt for copying. +# the install instructions use cpdup / /mnt, +# but using this second copy of the CD avoids all the tmpfs +# that are mounted on top of /, as well as our local modifications (like the gopherinstall user). +mount_cd9660 /dev/cd2 /mnt/mnt +cpdup /mnt/mnt/boot /mnt/boot +cpdup /mnt/mnt /mnt + +cat >/mnt/etc/rc.conf <<'EOF' +ifconfig_vtnet0='DHCP mtu 1460' +EOF + +cat >/mnt/boot/loader.conf <<'EOF' +vfs.root.mountfrom=hammer2:da0s1d +console=comconsole +EOF + +cat >/mnt/etc/fstab <<'EOF' +da0s1a /boot ufs rw 1 1 +da0s1d / hammer2 rw 1 1 +EOF + +umount /mnt/mnt +umount /mnt/boot + +echo 'DONE WITH PHASE 1.' +sync +poweroff +sleep 86400 diff --git a/env/dragonfly-amd64/phase2.sh b/env/dragonfly-amd64/phase2.sh new file mode 100644 index 0000000000..1a2070b258 --- /dev/null +++ b/env/dragonfly-amd64/phase2.sh @@ -0,0 +1,27 @@ +#!/bin/sh + +# Phase 2 of the DragonflyBSD installation: update pkg database. + +set -ex + +echo >&2 phase2.sh starting + +# Make pfi look for CD again when booting for phase3. +# Normally /etc/pfi.conf is left behind and stops future checks. +# Edit /etc/rc.d/pfi to remove /etc/pfi.conf each time it starts running. +echo '/REQUIRE/a +rm -f /etc/pfi.conf +. +w +q' | ed /etc/rc.d/pfi + +# pfi startup does not have full path that a root login does. +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/pkg/sbin:/usr/pkg/bin:/root/bin + +# Update pkg database and install extras we need. +pkg install -y bash curl git gdb + +echo 'DONE WITH PHASE 2.' +sync +poweroff +sleep 86400 diff --git a/env/dragonfly-amd64/phase3.sh b/env/dragonfly-amd64/phase3.sh new file mode 100644 index 0000000000..a9b5b9eff9 --- /dev/null +++ b/env/dragonfly-amd64/phase3.sh @@ -0,0 +1,47 @@ +#!/usr/local/bin/bash + +# Phase 3 of the DragonflyBSD installation: apply buildlet customizations. + +set -ex +set -o pipefail + +echo >&2 phase3.sh starting +rm -f /etc/pfi.conf + +# pfi startup does not have full path that a root login does. +export PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/games:/usr/local/sbin:/usr/local/bin:/usr/pkg/sbin:/usr/pkg/bin:/root/bin + +# Add a gopher user. +pw useradd gopher -g 0 -c 'Gopher Gopherson' -s /bin/sh + +# Disable keyboard console that won't exist on Google Cloud, +# to silence errors about trying to run getty on them. +# Serial console will remain enabled. +perl -pi -e 's/^ttyv/# ttyv/' /etc/ttys + +# Set up buildlet service. +cp /mnt/buildlet /etc/rc.d/buildlet +chmod +x /etc/rc.d/buildlet + +# Update rc.conf to run buildlet. +cat >>/etc/rc.conf <<'EOF' +hostname="buildlet" +rc_info="YES" +rc_startmsgs="YES" +sshd_enable="YES" +buildlet_enable="YES" +buildlet_env="PATH=/bin:/sbin:/usr/bin:/usr/local/bin" +EOF + +# Update loader.conf to disable checksum offload. +cat >>/boot/loader.conf <<'EOF' +hw.vtnet.csum_disable="1" +EOF + +# Generate ssh keys if needed. +service sshd keygen + +echo 'DONE WITH PHASE 3.' +sync +poweroff +sleep 86400 diff --git a/env/freebsd-amd64/buildlet b/env/freebsd-amd64/buildlet index 1772ec2323..f47ad35cfd 100755 --- a/env/freebsd-amd64/buildlet +++ b/env/freebsd-amd64/buildlet @@ -22,7 +22,6 @@ load_rc_config $name buildlet_prestart() { local buildlet_url - local retval info $(netstat -rn) info $(cat /etc/resolv.conf) @@ -30,9 +29,7 @@ buildlet_prestart() buildlet_url=$(/usr/local/bin/curl -s -H "Metadata-Flavor: Google" \ http://metadata.google.internal/computeMetadata/v1/instance/attributes/buildlet-binary-url) - /usr/local/bin/curl -s -o /buildlet "${buildlet_url}" >/dev/null 2>&1 - retval=$? - if [ $retval -ne 0 ]; then + if ! /usr/local/bin/curl -o /buildlet "${buildlet_url}"; then warn "failed to download buildlet from ${buildlet_url}" poweroff return 1