diff --git a/buildlet/gce.go b/buildlet/gce.go index e6bd499763..17555f3764 100644 --- a/buildlet/gce.go +++ b/buildlet/gce.go @@ -146,16 +146,7 @@ func StartNewVM(creds *google.Credentials, buildEnv *buildenv.Environment, instN if hconf.IsContainer() { if hconf.NestedVirt { minCPU = "Intel Haswell" // documented minimum from https://cloud.google.com/compute/docs/instances/enable-nested-virtualization-vm-instances - // TODO: use some variant of cosImage that finds our local - // forked copy of cos-stable with the VMX license added. For - // now, I just manually once ran: - // gcloud compute images create cos-stable-72-11316-136-0-vmx --source-image=cos-stable-72-11316-136-0 --source-image-project=cos-cloud --licenses=https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx - // And we'll use that version for now. Perhaps when Nested - // Virtualization reaches GA it'll just become a boolean we - // can set in our compute.Instance creation request and this - // license opt-in mechanism will be unnecessary. - const cosVMXImage = "cros-kvm-2" - srcImage = "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + cosVMXImage + srcImage = "https://www.googleapis.com/compute/v1/projects/" + projectID + "/global/images/" + hconf.ContainerVMImage() } else { var err error srcImage, err = cosImage(ctx, computeService) diff --git a/cmd/debugnewvm/debugnewvm.go b/cmd/debugnewvm/debugnewvm.go index 8075ed2ec2..0e84ea0265 100644 --- a/cmd/debugnewvm/debugnewvm.go +++ b/cmd/debugnewvm/debugnewvm.go @@ -79,9 +79,9 @@ func main() { hconf.VMImage = img } } - vmImageSummary := hconf.VMImage + vmImageSummary := fmt.Sprintf("%q", hconf.VMImage) if hconf.IsContainer() { - vmImageSummary = fmt.Sprintf("cos-stable, running %v", hconf.ContainerImage) + vmImageSummary = fmt.Sprintf("%q, running container %q", hconf.ContainerVMImage(), hconf.ContainerImage) } env = buildenv.FromFlags() @@ -96,7 +96,7 @@ func main() { name := fmt.Sprintf("debug-temp-%d", time.Now().Unix()) - log.Printf("Creating %s (with VM image %q)", name, vmImageSummary) + log.Printf("Creating %s (with VM image %s)", name, vmImageSummary) bc, err := buildlet.StartNewVM(creds, env, name, *hostType, buildlet.VMOpts{ OnInstanceRequested: func() { log.Printf("instance requested") }, OnInstanceCreated: func() { diff --git a/dashboard/builders.go b/dashboard/builders.go index 6400f484e3..1f6ff122cf 100644 --- a/dashboard/builders.go +++ b/dashboard/builders.go @@ -585,7 +585,8 @@ type HostConfig struct { HermeticReverse bool // whether reverse buildlet has fresh env per conn // Container image options, if ContainerImage != "": - NestedVirt bool // container requires VMX nested virtualization + NestedVirt bool // container requires VMX nested virtualization + KonletVMImage string // optional VM image (containing konlet) to use instead of default // Optional base env. GOROOT_BOOTSTRAP should go here if the buildlet // has Go 1.4+ baked in somewhere. @@ -1050,6 +1051,17 @@ func (c *HostConfig) PoolName() string { panic("unknown builder type") } +// ContainerVMImage returns the base VM name (not the fully qualified +// URL resource name of the VM) that starts the konlet program that +// pulls & runs a container. This method is only applicable when +// c.IsContainer() is true. +func (c *HostConfig) ContainerVMImage() string { + if c.KonletVMImage != "" { + return c.KonletVMImage + } + return "debian-stretch-vmx" +} + // IsHermetic reports whether this host config gets a fresh // environment (including /usr, /var, etc) for each execution. This is // true for VMs, GKE, and reverse buildlets running their containers diff --git a/env/linux-x86-vmx/README.md b/env/linux-x86-vmx/README.md new file mode 100644 index 0000000000..d8f21c1e6c --- /dev/null +++ b/env/linux-x86-vmx/README.md @@ -0,0 +1,6 @@ +# linux-x86-vmx + +These scripts create a GCE VM image that acts like Container-Optimized +Linux but uses a Debian 9 (Stretch) kernel + userspace instead. We do +this because Debian 9 includes CONFIG_KVM for nested virtualization, +whereas that's not compiled in for Container-Optimized Linux. diff --git a/env/linux-x86-vmx/create.sh b/env/linux-x86-vmx/create.sh new file mode 100755 index 0000000000..35cc0ab87a --- /dev/null +++ b/env/linux-x86-vmx/create.sh @@ -0,0 +1,76 @@ +#!/bin/sh +# Copyright 2019 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This creates the debian-stretch-vmx buildlet VM that's +# like the Container-Optimized OS but using Debian Stretch +# instead of the Chromium OS, and with nested virtualization +# enabled. + +set -e +set -x + +ZONE=us-central1-f +TARGET_IMAGE=debian-stretch-vmx + +TMP_DISK=dev-debian-vmx-tmpdisk +TMP_IMG=dev-debian-vmx-image +TMP_VM=dev-debian-vmx + +# Create disk, forking Debian 9 (Stretch). +gcloud compute disks delete $TMP_DISK --zone=$ZONE --quiet || true +gcloud compute disks create $TMP_DISK \ + --zone=$ZONE \ + --size=20GB \ + --image-project=debian-cloud \ + --image-family debian-9 + +# Create image based on that disk, with the nested virtualization +# opt-in flag ("license"). +gcloud compute images delete $TMP_IMG --quiet || true +gcloud compute images create \ + $TMP_IMG \ + --source-disk=$TMP_DISK \ + --source-disk-zone=$ZONE \ + --licenses "https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx" + +# No longer need that temp disk: +gcloud compute disks delete $TMP_DISK --zone=$ZONE --quiet + +# Create the VM +gcloud compute instances delete --zone=$ZONE $TMP_VM --quiet || true +gcloud compute instances create \ + $TMP_VM \ + --zone=$ZONE \ + --image=$TMP_IMG \ + --min-cpu-platform "Intel Haswell" + +INTERNAL_IP=$(gcloud --format="value(networkInterfaces[0].networkIP)" compute instances list --filter="name=('$TMP_VM')") +EXTERNAL_IP=$(gcloud --format="value(networkInterfaces[0].accessConfigs[0].natIP)" compute instances list --filter="name=('$TMP_VM')") +echo "external IP: $EXTERNAL_IP, internal IP: $INTERNAL_IP" + +echo "Waiting for SSH port to be available..." +while ! nc -w 2 -z $INTERNAL_IP 22; do + sleep 1 +done + +echo "SSH is up. Copying prep-vm.sh script to VM..." + +# gcloud compute scp lacks an --internal-ip flag, even though gcloud +# compute ssh has it. Annoying. Workaround: +gcloud compute scp --dry-run --zone=$ZONE prep-vm.sh bradfitz@$TMP_VM: | perl -npe "s/$EXTERNAL_IP/$INTERNAL_IP/" | sh + +# And prep the machine. +gcloud compute ssh $TMP_VM --zone=$ZONE --internal-ip -- sudo bash ./prep-vm.sh + +echo "Done prepping machine; shutting down" + +# Shut it down so it's a stable source to snapshot from. +gcloud compute instances stop $TMP_VM --zone=$ZONE + +# Now make the new image from our instance's disk. +gcloud compute images delete $TARGET_IMAGE --quiet || true +gcloud compute images create $TARGET_IMAGE --source-disk=$TMP_VM --source-disk-zone=$ZONE + +echo "Done." diff --git a/env/linux-x86-vmx/prep-vm.sh b/env/linux-x86-vmx/prep-vm.sh new file mode 100644 index 0000000000..2edd7892d6 --- /dev/null +++ b/env/linux-x86-vmx/prep-vm.sh @@ -0,0 +1,34 @@ +#!/bin/false +# Copyright 2019 The Go Authors. All rights reserved. +# Use of this source code is governed by a BSD-style +# license that can be found in the LICENSE file. + +# This runs on the Debian Stretch template VM to turn it into the +# buildlet image we want. This isn't for running on the developer's +# host machine. + +set -e +set -x + +apt-get update +apt-get install --yes apt-transport-https ca-certificates curl gnupg2 software-properties-common +curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add - +add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian $(lsb_release -cs) stable" +apt-get update +apt-get install --yes docker-ce docker-ce-cli containerd.io + +git clone https://github.com/GoogleCloudPlatform/konlet.git +mkdir -p /usr/share/google +install konlet/scripts/get_metadata_value /usr/share/google +mkdir -p /usr/share/gce-containers +install konlet/scripts/konlet-startup /usr/share/gce-containers/konlet-startup +install konlet/scripts/konlet-startup.service /etc/systemd/system +systemctl enable /etc/systemd/system/konlet-startup.service +systemctl start konlet-startup + +# Pre-pull some common images/layers to speed up future boots: +gcloud auth configure-docker --quiet +docker pull gcr.io/symbolic-datum-552/linux-x86-stretch:latest +docker pull gcr.io/gce-containers/konlet:v.0.9-latest + +apt-get dist-upgrade --yes