Skip to content

Commit

Permalink
cmd/rundockerbuildlet, buildlet: enable running arm EC2 instances
Browse files Browse the repository at this point in the history 
This enables rundockerbuildlet to run non-reverse buildlet
image on EC2. It will only run a single instance of rundockerbuildlet
once. It exposes port 443 for the coordinator to authenticate with the running
buildlet.

This also adds the buildlet name and buildlet container URL to
the EC2 user data struct retrieved by rundockerbuildlet.

Updates golang/go#36841

Change-Id: I31de754e2ac8970c6f18993104de0e0baea5dc31
Reviewed-on: https://go-review.googlesource.com/c/build/+/234114
Run-TryBot: Carlos Amedee <carlos@golang.org>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Dmitri Shuralyov <dmitshur@golang.org>
  • Loading branch information
@cagedmantis
cagedmantis committed May 15, 2020
1 parent d1deecf commit 92f0c56
Show file tree
Hide file tree
Showing 3 changed files with 122 additions and 28 deletions.
6 changes: 5 additions & 1 deletion buildlet/aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ import (
type EC2UserData struct {
BuildletBinaryURL string `json:"buildlet_binary_url,omitempty"`
BuildletHostType string `json:"buildlet_host_type,omitempty"`
BuildletImageURL string `json:"buildlet_image_url,omitempty"`
BuildletName string `json:"buildlet_name,omitempty"`
Metadata map[string]string `json:"metadata,omitempty"`
TLSCert string `json:"tls_cert,omitempty"`
TLSKey string `json:"tls_key,omitempty"`
Expand Down Expand Up @@ -184,12 +186,14 @@ func (c *AWSClient) configureVM(buildEnv *buildenv.Environment, hconf *dashboard
func (c *AWSClient) vmUserDataSpec(vmConfig *ec2.RunInstancesInput, buildEnv *buildenv.Environment, hconf *dashboard.HostConfig, vmName, hostType string, opts *VMOpts) {
// add custom metadata to the user data.
ud := EC2UserData{
BuildletName: vmName,
BuildletBinaryURL: hconf.BuildletBinaryURL(buildEnv),
BuildletHostType: hostType,
BuildletImageURL: hconf.ContainerVMImage(),
Metadata: make(map[string]string),
TLSCert: opts.TLS.CertPEM,
TLSKey: opts.TLS.KeyPEM,
TLSPassword: opts.TLS.Password(),
Metadata: make(map[string]string),
}
for k, v := range opts.Meta {
ud.Metadata[k] = v
Expand Down
66 changes: 41 additions & 25 deletions buildlet/aws_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -432,33 +432,40 @@ func TestEC2BuildletParams(t *testing.T) {

func TestConfigureVM(t *testing.T) {
testCases := []struct {
desc string
buildEnv *buildenv.Environment
hconf *dashboard.HostConfig
hostType string
opts *VMOpts
vmName string
wantDesc string
wantImageID string
wantInstanceType string
wantName string
wantZone string
desc string
buildEnv *buildenv.Environment
hconf *dashboard.HostConfig
hostType string
opts *VMOpts
vmName string
wantDesc string
wantImageID string
wantInstanceType string
wantName string
wantZone string
wantBuildletName string
wantBuildletImage string
}{
{
desc: "default-values",
buildEnv: &buildenv.Environment{},
hconf: &dashboard.HostConfig{},
vmName: "base_vm",
hostType: "host-foo-bar",
opts: &VMOpts{},
wantInstanceType: "n1-highcpu-2",
wantName: "base_vm",
desc: "default-values",
buildEnv: &buildenv.Environment{},
hconf: &dashboard.HostConfig{
KonletVMImage: "gcr.io/symbolic-datum-552/gobuilder-arm64-aws",
},
vmName: "base_vm",
hostType: "host-foo-bar",
opts: &VMOpts{},
wantInstanceType: "n1-highcpu-2",
wantName: "base_vm",
wantBuildletName: "base_vm",
wantBuildletImage: "gcr.io/symbolic-datum-552/gobuilder-arm64-aws",
},
{
desc: "full-configuration",
buildEnv: &buildenv.Environment{},
hconf: &dashboard.HostConfig{
VMImage: "awesome_image",
VMImage: "awesome_image",
KonletVMImage: "gcr.io/symbolic-datum-552/gobuilder-arm64-aws",
},
vmName: "base-vm",
hostType: "host-foo-bar",
Expand All @@ -473,11 +480,13 @@ func TestConfigureVM(t *testing.T) {
"sample": "value",
},
},
wantDesc: "test description",
wantImageID: "awesome_image",
wantInstanceType: "n1-highcpu-2",
wantName: "base-vm",
wantZone: "sa-west",
wantDesc: "test description",
wantImageID: "awesome_image",
wantInstanceType: "n1-highcpu-2",
wantName: "base-vm",
wantZone: "sa-west",
wantBuildletName: "base-vm",
wantBuildletImage: "gcr.io/symbolic-datum-552/gobuilder-arm64-aws",
},
}
for _, tc := range testCases {
Expand Down Expand Up @@ -533,6 +542,13 @@ func TestConfigureVM(t *testing.T) {
if gotUD.BuildletHostType != tc.hostType {
t.Errorf("buildletHostType got %s; want %s", gotUD.BuildletHostType, tc.hostType)
}
if gotUD.BuildletName != tc.wantBuildletName {
t.Errorf("buildletName got %s; want %s", gotUD.BuildletName, tc.wantBuildletName)
}
if gotUD.BuildletImageURL != tc.wantBuildletImage {
t.Errorf("buildletImageURL got %s; want %s", gotUD.BuildletImageURL, tc.wantBuildletImage)
}

if gotUD.TLSCert != tc.opts.TLS.CertPEM {
t.Errorf("TLSCert got %s; want %s", gotUD.TLSCert, tc.opts.TLS.CertPEM)
}
Expand Down
78 changes: 76 additions & 2 deletions cmd/rundockerbuildlet/rundockerbuildlet.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ package main

import (
"bytes"
"context"
"encoding/json"
"flag"
"fmt"
Expand All @@ -23,7 +24,10 @@ import (
"strings"
"time"

"github.com/aws/aws-sdk-go/aws/ec2metadata"
"github.com/aws/aws-sdk-go/aws/session"
"golang.org/x/build/buildenv"
"golang.org/x/build/buildlet"
)

var (
Expand All @@ -40,6 +44,12 @@ var (
var (
buildKey []byte
scalewayMeta = new(scalewayMetadata)
isReverse = true
isSingleRun = false
// ec2UD contains a copy of the EC2 vm user data retrieved from the metadata.
ec2UD *buildlet.EC2UserData
// ec2MetaClient is an EC2 metadata client.
ec2MetaClient *ec2metadata.EC2Metadata
)

func main() {
Expand All @@ -52,9 +62,19 @@ func main() {
*numInst = 1
*basename = "scaleway"
initScalewayMeta()
} else if onEC2() {
initEC2Meta()
*memory = ""
*image = ec2UD.BuildletImageURL
*pull = true
*numInst = 1
isReverse = false
isSingleRun = true
}

buildKey = getBuildKey()
if isReverse {
buildKey = getBuildKey()
}

if *image == "" {
log.Fatalf("docker --image is required")
Expand All @@ -65,10 +85,35 @@ func main() {
if err := checkFix(); err != nil {
log.Print(err)
}
if isSingleRun {
log.Printf("Configured to run a single instance. Exiting")
os.Exit(0)
}
time.Sleep(time.Second) // TODO: docker wait on the running containers?
}
}

func ec2MdClient() *ec2metadata.EC2Metadata {
if ec2MetaClient != nil {
return ec2MetaClient
}
ses, err := session.NewSession()
if err != nil {
return nil
}
ec2MetaClient = ec2metadata.New(ses)
return ec2MetaClient
}

func onEC2() bool {
if ec2MdClient() == nil {
return false
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return ec2MdClient().AvailableWithContext(ctx)
}

func onScaleway() bool {
if *builderEnv == "host-linux-arm-scaleway" {
return true
Expand Down Expand Up @@ -136,6 +181,8 @@ func checkFix() error {
// c1 instance hostname for debugability.
// There should only be one running container per c1 instance.
name = scalewayMeta.Hostname
} else if onEC2() {
name = ec2UD.BuildletName
} else {
name = fmt.Sprintf("%s%02d", *basename, num)
}
Expand Down Expand Up @@ -185,13 +232,17 @@ func checkFix() error {
cmd := exec.Command("docker", "run",
"-d",
"--name="+name,
"-v", filepath.Dir(keyFile)+":/buildkey/",
"-e", "HOSTNAME="+name,
"--security-opt=seccomp=unconfined", // Issue 35547
"--tmpfs=/workdir:rw,exec")
if *memory != "" {
cmd.Args = append(cmd.Args, "--memory="+*memory)
}
if isReverse {
cmd.Args = append(cmd.Args, "-v", filepath.Dir(keyFile)+":/buildkey/")
} else {
cmd.Args = append(cmd.Args, "-p", "443:443")
}
if *cpu > 0 {
cmd.Args = append(cmd.Args, fmt.Sprintf("--cpuset-cpus=%d-%d", *cpu*(num-1), *cpu*num-1))
}
Expand Down Expand Up @@ -234,6 +285,29 @@ func (m *scalewayMetadata) HasTag(t string) bool {
return false
}

func initEC2Meta() {
if !onEC2() {
log.Fatal("attempt to initialize metadata on non-EC2 instance")
}
if ec2UD != nil {
return
}
if ec2MdClient() == nil {
log.Fatalf("unable to retrieve EC2 metadata client")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
ec2MetaJson, err := ec2MdClient().GetUserDataWithContext(ctx)
if err != nil {
log.Fatalf("unable to retrieve EC2 user data: %v", err)
}
ec2UD = &buildlet.EC2UserData{}
err = json.Unmarshal([]byte(ec2MetaJson), ec2UD)
if err != nil {
log.Fatalf("unable to unmarshal user data json: %v", err)
}
}

func initScalewayMeta() {
const metaURL = "http://169.254.42.42/conf?format=json"
res, err := http.Get(metaURL)
Expand Down

0 comments on commit 92f0c56

Please sign in to comment.