diff --git a/buildenv/envs.go b/buildenv/envs.go index f80a186bee..e54e3b930e 100644 --- a/buildenv/envs.go +++ b/buildenv/envs.go @@ -74,11 +74,18 @@ type Environment struct { // other fields. ControlZone string + // PreferredAvailabilityZone is the preffered AWS availability zone. + PreferredAvailabilityZone string + // VMZones are the GCE zones that the VMs will be deployed to. These // GCE zones will be periodically cleaned by deleting old VMs. The zones // should all exist within a single region. VMZones []string + // VMAvailabilityZones are the AWS availability zones that the VMs will be deployed to. + // The availability zones should all exist within a single region. + VMAvailabilityZones []string + // StaticIP is the public, static IP address that will be attached to the // coordinator instance. The zero value means the address will be looked // up by name. This field is optional. @@ -151,6 +158,16 @@ func (e Environment) RandomVMZone() string { return e.VMZones[rand.Intn(len(e.VMZones))] } +// RandomAWSVMZone returns a randomly selected zone from the zones in +// VMAvailabilityZones. The PreferredAvailabilityZone value will be +// returned if VMAvailabilityZones is not set. +func (e Environment) RandomAWSVMZone() string { + if len(e.VMAvailabilityZones) == 0 { + return e.PreferredAvailabilityZone + } + return e.VMAvailabilityZones[rand.Intn(len(e.VMZones))] +} + // Region returns the GCE region, derived from its zone. func (e Environment) Region() string { return e.ControlZone[:strings.LastIndex(e.ControlZone, "-")] diff --git a/buildlet/aws.go b/buildlet/aws.go new file mode 100644 index 0000000000..a24aca50b7 --- /dev/null +++ b/buildlet/aws.go @@ -0,0 +1,241 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildlet + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "time" + + "golang.org/x/build/buildenv" + "golang.org/x/build/dashboard" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" +) + +// AWSUserData is stored in the user data for each EC2 instance. This is +// used to store metadata about the running instance. The buildlet will retrieve +// this on EC2 instances before allowing connections from the coordinator. +type AWSUserData struct { + BuildletBinaryURL string `json:"buildlet_binary_url,omitempty"` + BuildletHostType string `json:"buildlet_host_type,omitempty"` + Metadata map[string]string `json:"metadata,omitempty"` + TLSCert string `json:"tls_cert,omitempty"` + TLSKey string `json:"tls_key,omitempty"` + TLSPassword string `json:"tls_password,omitempty"` +} + +// ec2Client represents the EC2 specific calls made durring the +// lifecycle of a buildlet. +type ec2Client interface { + DescribeInstancesWithContext(context.Context, *ec2.DescribeInstancesInput, ...request.Option) (*ec2.DescribeInstancesOutput, error) + RunInstancesWithContext(context.Context, *ec2.RunInstancesInput, ...request.Option) (*ec2.Reservation, error) + TerminateInstancesWithContext(context.Context, *ec2.TerminateInstancesInput, ...request.Option) (*ec2.TerminateInstancesOutput, error) + WaitUntilInstanceExistsWithContext(context.Context, *ec2.DescribeInstancesInput, ...request.WaiterOption) error +} + +// AWSClient is the client used to create and destroy buildlets on AWS. +type AWSClient struct { + client ec2Client +} + +// NewAWSClient creates a new AWSClient. +func NewAWSClient(region, keyID, accessKey string) (*AWSClient, error) { + s, err := session.NewSession(&aws.Config{ + Region: aws.String(region), + Credentials: credentials.NewStaticCredentials(keyID, accessKey, ""), // Token is only required for STS + }) + if err != nil { + return nil, fmt.Errorf("failed to create AWS session: %v", err) + } + return &AWSClient{ + client: ec2.New(s), + }, nil +} + +// StartNewVM boots a new VM on EC2, waits until the client is accepting connections +// on the configured port and returns a buildlet client configured communicate with it. +func (c *AWSClient) StartNewVM(ctx context.Context, buildEnv *buildenv.Environment, hconf *dashboard.HostConfig, vmName, hostType string, opts *VMOpts) (*Client, error) { + // check required params + if opts == nil || opts.TLS.IsZero() { + return nil, errors.New("TLS keypair is not set") + } + if buildEnv == nil { + return nil, errors.New("invalid build enviornment") + } + if hconf == nil { + return nil, errors.New("invalid host configuration") + } + if vmName == "" || hostType == "" { + return nil, fmt.Errorf("invalid vmName: %q and hostType: %q", vmName, hostType) + } + + // configure defaults + if opts.Description == "" { + opts.Description = fmt.Sprintf("Go Builder for %s", hostType) + } + if opts.Zone == "" { + opts.Zone = buildEnv.RandomAWSVMZone() + } + if opts.DeleteIn == 0 { + opts.DeleteIn = 30 * time.Minute + } + + vmConfig := c.configureVM(buildEnv, hconf, vmName, hostType, opts) + vmID, err := c.createVM(ctx, vmConfig, opts) + if err != nil { + return nil, err + } + if err = c.WaitUntilVMExists(ctx, vmID, opts); err != nil { + return nil, err + } + vm, err := c.RetrieveVMInfo(ctx, vmID) + if err != nil { + return nil, err + } + buildletURL, ipPort, err := ec2BuildletParams(vm, opts) + if err != nil { + return nil, err + } + return buildletClient(ctx, buildletURL, ipPort, opts) +} + +// createVM submits a request for the creation of a VM. +func (c *AWSClient) createVM(ctx context.Context, vmConfig *ec2.RunInstancesInput, opts *VMOpts) (string, error) { + runResult, err := c.client.RunInstancesWithContext(ctx, vmConfig) + if err != nil { + return "", fmt.Errorf("unable to create instance: %w", err) + } + condRun(opts.OnInstanceRequested) + return *runResult.Instances[0].InstanceId, nil +} + +// WaitUntilVMExists submits a request which waits until an instance exists before returning. +func (c *AWSClient) WaitUntilVMExists(ctx context.Context, instID string, opts *VMOpts) error { + err := c.client.WaitUntilInstanceExistsWithContext(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instID)}, + }) + if err != nil { + return fmt.Errorf("failed waiting for vm instance: %w", err) + } + condRun(opts.OnInstanceCreated) + return err +} + +// RetrieveVMInfo retrives the information about a VM. +func (c *AWSClient) RetrieveVMInfo(ctx context.Context, instID string) (*ec2.Instance, error) { + instances, err := c.client.DescribeInstancesWithContext(ctx, &ec2.DescribeInstancesInput{ + InstanceIds: []*string{aws.String(instID)}, + }) + if err != nil { + return nil, fmt.Errorf("unable to retrieve instance %q information: %w", instID, err) + } + + instance, err := ec2Instance(instances) + if err != nil { + return nil, fmt.Errorf("failed to read instance description: %w", err) + } + return instance, err +} + +// configureVM creates a configuration for an EC2 VM instance. +func (c *AWSClient) configureVM(buildEnv *buildenv.Environment, hconf *dashboard.HostConfig, vmName, hostType string, opts *VMOpts) *ec2.RunInstancesInput { + vmConfig := &ec2.RunInstancesInput{ + ImageId: aws.String(hconf.VMImage), + InstanceType: aws.String(hconf.MachineType()), + MinCount: aws.Int64(1), + MaxCount: aws.Int64(1), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String(opts.Zone), + }, + InstanceInitiatedShutdownBehavior: aws.String("terminate"), + TagSpecifications: []*ec2.TagSpecification{ + &ec2.TagSpecification{ + Tags: []*ec2.Tag{ + &ec2.Tag{ + Key: aws.String("Name"), + Value: aws.String(vmName), + }, + &ec2.Tag{ + Key: aws.String("Description"), + Value: aws.String(opts.Description), + }, + }, + }, + }, + } + + // add custom metadata to the user data. + ud := AWSUserData{ + BuildletBinaryURL: hconf.BuildletBinaryURL(buildEnv), + BuildletHostType: hostType, + TLSCert: opts.TLS.CertPEM, + TLSKey: opts.TLS.KeyPEM, + TLSPassword: opts.TLS.Password(), + Metadata: make(map[string]string), + } + for k, v := range opts.Meta { + ud.Metadata[k] = v + } + jsonUserData, err := json.Marshal(ud) + if err != nil { + log.Printf("unable to marshal user data: %v", err) + } + return vmConfig.SetUserData(string(jsonUserData)) +} + +// DestroyVM submits a request to destroy a VM. +func (c *AWSClient) DestroyVM(ctx context.Context, vmID string) error { + _, err := c.client.TerminateInstancesWithContext(ctx, &ec2.TerminateInstancesInput{ + InstanceIds: []*string{aws.String(vmID)}, + }) + if err != nil { + return fmt.Errorf("unable to destroy vm: %w", err) + } + return err +} + +// ec2Instance extracts the first instance found in the the describe instances output. +func ec2Instance(dio *ec2.DescribeInstancesOutput) (*ec2.Instance, error) { + if dio == nil || dio.Reservations == nil || dio.Reservations[0].Instances == nil { + return nil, errors.New("describe instances output does not contain a valid instance") + } + return dio.Reservations[0].Instances[0], nil +} + +// ec2InstanceIPs returns the internal and external ip addresses for the VM. +func ec2InstanceIPs(inst *ec2.Instance) (intIP, extIP string, err error) { + if inst.PrivateIpAddress == nil || *inst.PrivateIpAddress == "" { + return "", "", errors.New("internal IP address is not set") + } + if inst.PublicIpAddress == nil || *inst.PublicIpAddress == "" { + return "", "", errors.New("external IP address is not set") + } + return *inst.PrivateIpAddress, *inst.PublicIpAddress, nil +} + +// ec2BuildletParams returns the necessary information to connect to an EC2 buildlet. A +// buildlet URL and an IP address port are required to connect to a buildlet. +func ec2BuildletParams(inst *ec2.Instance, opts *VMOpts) (string, string, error) { + _, extIP, err := ec2InstanceIPs(inst) + if err != nil { + return "", "", fmt.Errorf("failed to retrieve IP addresses: %w", err) + } + buildletURL := fmt.Sprintf("https://%s", extIP) + ipPort := net.JoinHostPort(extIP, "443") + + if opts.OnGotEC2InstanceInfo != nil { + opts.OnGotEC2InstanceInfo(inst) + } + return buildletURL, ipPort, err +} diff --git a/buildlet/aws_test.go b/buildlet/aws_test.go new file mode 100644 index 0000000000..26d3537642 --- /dev/null +++ b/buildlet/aws_test.go @@ -0,0 +1,717 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildlet + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/request" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/google/go-cmp/cmp" + "golang.org/x/build/buildenv" + "golang.org/x/build/dashboard" +) + +type fakeEC2Client struct { + // returned in describe instances + PrivateIP *string + PublicIP *string +} + +func (f *fakeEC2Client) DescribeInstancesWithContext(ctx context.Context, input *ec2.DescribeInstancesInput, opt ...request.Option) (*ec2.DescribeInstancesOutput, error) { + if ctx == nil || input == nil || len(input.InstanceIds) == 0 { + return nil, request.ErrInvalidParams{} + } + return &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + &ec2.Reservation{ + Instances: []*ec2.Instance{ + &ec2.Instance{ + InstanceId: input.InstanceIds[0], + PrivateIpAddress: f.PrivateIP, + PublicIpAddress: f.PublicIP, + }, + }, + }, + }, + }, nil +} + +func (f *fakeEC2Client) RunInstancesWithContext(ctx context.Context, input *ec2.RunInstancesInput, opts ...request.Option) (*ec2.Reservation, error) { + if ctx == nil || input == nil { + return nil, request.ErrInvalidParams{} + } + if input.ImageId == nil || input.InstanceType == nil || input.MinCount == nil || input.Placement == nil { + return nil, errors.New("invalid instance configuration") + } + return &ec2.Reservation{ + Instances: []*ec2.Instance{ + &ec2.Instance{ + ImageId: input.ImageId, + InstanceType: input.InstanceType, + InstanceId: aws.String("44"), + Placement: input.Placement, + }, + }, + ReservationId: aws.String("res_id"), + }, nil +} + +func (f *fakeEC2Client) TerminateInstancesWithContext(ctx context.Context, input *ec2.TerminateInstancesInput, opts ...request.Option) (*ec2.TerminateInstancesOutput, error) { + if ctx == nil || input == nil || len(input.InstanceIds) == 0 { + return nil, request.ErrInvalidParams{} + } + for _, id := range input.InstanceIds { + if *id == "" { + return nil, errors.New("invalid instance id") + } + } + return &ec2.TerminateInstancesOutput{ + TerminatingInstances: nil, + }, nil +} + +func (f *fakeEC2Client) WaitUntilInstanceExistsWithContext(ctx context.Context, input *ec2.DescribeInstancesInput, opt ...request.WaiterOption) error { + if ctx == nil || input == nil || len(input.InstanceIds) == 0 { + return request.ErrInvalidParams{} + } + return nil +} + +func TestRetrieveVMInfo(t *testing.T) { + wantVMID := "22" + ctx := context.Background() + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotInst, gotErr := c.RetrieveVMInfo(ctx, wantVMID) + if gotErr != nil { + t.Fatalf("RetrieveVMInfo(%v, %q) failed with error %s", ctx, wantVMID, gotErr) + } + if gotInst == nil || *gotInst.InstanceId != wantVMID { + t.Errorf("RetrieveVMInfo(%v, %q) failed with error %s", ctx, wantVMID, gotErr) + } +} + +func TestStartNewVM(t *testing.T) { + kp, err := NewKeyPair() + if err != nil { + t.Fatalf("unable to generate key pair: %s", err) + } + buildEnv := &buildenv.Environment{} + hconf := &dashboard.HostConfig{} + vmName := "sample-vm" + hostType := "host-sample-os" + opts := &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + TLS: kp, + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + SkipEndpointVerification: true, + } + c := &AWSClient{ + client: &fakeEC2Client{ + PrivateIP: aws.String("8.8.8.8"), + PublicIP: aws.String("9.9.9.9"), + }, + } + gotClient, gotErr := c.StartNewVM(context.Background(), buildEnv, hconf, vmName, hostType, opts) + if gotErr != nil { + t.Fatalf("error is not nil: %v", gotErr) + } + if gotClient == nil { + t.Fatalf("response is nil") + } +} + +func TestStartNewVMError(t *testing.T) { + kp, err := NewKeyPair() + if err != nil { + t.Fatalf("unable to generate key pair: %s", err) + } + + testCases := []struct { + desc string + buildEnv *buildenv.Environment + hconf *dashboard.HostConfig + vmName string + hostType string + opts *VMOpts + }{ + { + desc: "nil-buildenv", + hconf: &dashboard.HostConfig{}, + vmName: "sample-vm", + hostType: "host-sample-os", + opts: &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + TLS: kp, + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + }, + }, + { + desc: "nil-hconf", + buildEnv: &buildenv.Environment{}, + vmName: "sample-vm", + hostType: "host-sample-os", + opts: &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + TLS: kp, + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + }, + }, + { + desc: "empty-vnName", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{}, + vmName: "", + hostType: "host-sample-os", + opts: &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + TLS: kp, + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + }, + }, + { + desc: "empty-hostType", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{}, + vmName: "sample-vm", + hostType: "", + opts: &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + TLS: kp, + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + }, + }, + { + desc: "missing-certs", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{}, + vmName: "sample-vm", + hostType: "host-sample-os", + opts: &VMOpts{ + Zone: "us-west", + ProjectID: "project1", + Description: "Golang builder for sample", + Meta: map[string]string{ + "Owner": "george", + }, + DeleteIn: 45 * time.Second, + }, + }, + { + desc: "nil-opts", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{}, + vmName: "sample-vm", + hostType: "host-sample-os", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotClient, gotErr := c.StartNewVM(context.Background(), tc.buildEnv, tc.hconf, tc.vmName, tc.hostType, tc.opts) + if gotErr == nil { + t.Errorf("expected error did not occur") + } + if gotClient != nil { + t.Errorf("got %+v; expected nil", gotClient) + } + }) + } +} + +func TestWaitUntilInstanceExists(t *testing.T) { + vmID := "22" + invoked := false + opts := &VMOpts{ + OnInstanceCreated: func() { + invoked = true + }, + } + ctx := context.Background() + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotErr := c.WaitUntilVMExists(ctx, vmID, opts) + if gotErr != nil { + t.Fatalf("WaitUntilVMExists(%v, %v, %v) failed with error %s", ctx, vmID, opts, gotErr) + } + if !invoked { + t.Errorf("OnInstanceCreated() was not invoked") + } +} + +func TestCreateVM(t *testing.T) { + vmConfig := &ec2.RunInstancesInput{ + ImageId: aws.String("foo"), + InstanceType: aws.String("type-a"), + MinCount: aws.Int64(15), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("eu-15"), + }, + } + invoked := false + opts := &VMOpts{ + OnInstanceRequested: func() { + invoked = true + }, + } + wantVMID := aws.String("44") + + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotVMID, gotErr := c.createVM(context.Background(), vmConfig, opts) + if gotErr != nil { + t.Fatalf("createVM(ctx, %v, %v) failed with %s", vmConfig, opts, gotErr) + } + if gotVMID != *wantVMID { + t.Errorf("createVM(ctx, %v, %v) = %s, nil; want %s, nil", vmConfig, opts, gotVMID, *wantVMID) + } + if !invoked { + t.Errorf("OnInstanceRequested() was not invoked") + } +} + +func TestCreateVMError(t *testing.T) { + testCases := []struct { + desc string + vmConfig *ec2.RunInstancesInput + opts *VMOpts + }{ + { + desc: "missing-vmConfig", + }, + { + desc: "missing-image-id", + vmConfig: &ec2.RunInstancesInput{ + InstanceType: aws.String("type-a"), + MinCount: aws.Int64(15), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("eu-15"), + }, + }, + opts: &VMOpts{ + OnInstanceRequested: func() {}, + }, + }, + { + desc: "missing-instance-id", + vmConfig: &ec2.RunInstancesInput{ + ImageId: aws.String("foo"), + MinCount: aws.Int64(15), + Placement: &ec2.Placement{ + AvailabilityZone: aws.String("eu-15"), + }, + }, + opts: &VMOpts{ + OnInstanceRequested: func() {}, + }, + }, + { + desc: "missing-placement", + vmConfig: &ec2.RunInstancesInput{ + ImageId: aws.String("foo"), + InstanceType: aws.String("type-a"), + MinCount: aws.Int64(15), + }, + opts: &VMOpts{ + OnInstanceRequested: func() {}, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotVMID, gotErr := c.createVM(context.Background(), tc.vmConfig, tc.opts) + if gotErr == nil { + t.Errorf("createVM(ctx, %v, %v) = %s, %v; want error", tc.vmConfig, tc.opts, gotVMID, gotErr) + } + if gotVMID != "" { + t.Errorf("createVM(ctx, %v, %v) = %s, %v; %q, error", tc.vmConfig, tc.opts, gotVMID, gotErr, "") + } + }) + } +} + +func TestDestroyVM(t *testing.T) { + testCases := []struct { + desc string + ctx context.Context + vmID string + wantErr bool + }{ + {"baseline request", context.Background(), "vm-20", false}, + {"nil context", nil, "vm-20", true}, + {"nil context", context.Background(), "", true}, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := &AWSClient{ + client: &fakeEC2Client{}, + } + gotErr := c.DestroyVM(tc.ctx, tc.vmID) + if (gotErr != nil) != tc.wantErr { + t.Errorf("DestroyVM(%v, %q) = %v; want error %t", tc.ctx, tc.vmID, gotErr, tc.wantErr) + } + }) + } +} + +func TestEC2BuildletParams(t *testing.T) { + testCases := []struct { + desc string + inst *ec2.Instance + opts *VMOpts + wantURL string + wantPort string + wantCalled bool + }{ + { + desc: "base case", + inst: &ec2.Instance{ + PrivateIpAddress: aws.String("9.9.9.9"), + PublicIpAddress: aws.String("8.8.8.8"), + }, + opts: &VMOpts{}, + wantCalled: true, + wantURL: "https://8.8.8.8", + wantPort: "8.8.8.8:443", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + gotURL, gotPort, gotErr := ec2BuildletParams(tc.inst, tc.opts) + if gotErr != nil { + t.Fatalf("ec2BuildletParams(%v, %v) failed; %v", tc.inst, tc.opts, gotErr) + } + if gotURL != tc.wantURL || gotPort != tc.wantPort { + t.Errorf("ec2BuildletParams(%v, %v) = %q, %q, nil; want %q, %q, nil", tc.inst, tc.opts, gotURL, gotPort, tc.wantURL, tc.wantPort) + } + }) + } +} + +func TestConfigureVM(t *testing.T) { + testCases := []struct { + desc string + buildEnv *buildenv.Environment + hconf *dashboard.HostConfig + hostType string + opts *VMOpts + vmName string + wantDesc string + wantImageID string + wantInstanceType string + wantName string + wantZone string + }{ + { + desc: "default-values", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{}, + vmName: "base_vm", + hostType: "host-foo-bar", + opts: &VMOpts{}, + wantInstanceType: "n1-highcpu-2", + wantName: "base_vm", + }, + { + desc: "full-configuration", + buildEnv: &buildenv.Environment{}, + hconf: &dashboard.HostConfig{ + VMImage: "awesome_image", + }, + vmName: "base-vm", + hostType: "host-foo-bar", + opts: &VMOpts{ + Zone: "sa-west", + TLS: KeyPair{ + CertPEM: "abc", + KeyPEM: "xyz", + }, + Description: "test description", + Meta: map[string]string{ + "sample": "value", + }, + }, + wantDesc: "test description", + wantImageID: "awesome_image", + wantInstanceType: "n1-highcpu-2", + wantName: "base-vm", + wantZone: "sa-west", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + c := &AWSClient{} + got := c.configureVM(tc.buildEnv, tc.hconf, tc.vmName, tc.hostType, tc.opts) + if *got.ImageId != tc.wantImageID { + t.Errorf("ImageId got %s; want %s", *got.ImageId, tc.wantImageID) + } + if *got.InstanceType != tc.wantInstanceType { + t.Errorf("InstanceType got %s; want %s", *got.InstanceType, tc.wantInstanceType) + } + + if *got.MinCount != 1 { + t.Errorf("MinCount got %d; want %d", *got.MinCount, 1) + } + if *got.MaxCount != 1 { + t.Errorf("MaxCount got %d; want %d", *got.MaxCount, 1) + } + if *got.Placement.AvailabilityZone != tc.wantZone { + t.Errorf("AvailabilityZone got %s; want %s", *got.Placement.AvailabilityZone, tc.wantZone) + } + if *got.InstanceInitiatedShutdownBehavior != "terminate" { + t.Errorf("InstanceType got %s; want %s", *got.InstanceInitiatedShutdownBehavior, "terminate") + } + if *got.TagSpecifications[0].Tags[0].Key != "Name" { + t.Errorf("First Tag Key got %s; want %s", *got.TagSpecifications[0].Tags[0].Key, "Name") + } + if *got.TagSpecifications[0].Tags[0].Value != tc.wantName { + t.Errorf("First Tag Value got %s; want %s", *got.TagSpecifications[0].Tags[0].Value, tc.wantName) + } + if *got.TagSpecifications[0].Tags[1].Key != "Description" { + t.Errorf("Second Tag Key got %s; want %s", *got.TagSpecifications[0].Tags[1].Key, "Description") + } + if *got.TagSpecifications[0].Tags[1].Value != tc.wantDesc { + t.Errorf("Second Tag Value got %s; want %s", *got.TagSpecifications[0].Tags[1].Value, tc.wantDesc) + } + gotUD := &AWSUserData{} + err := json.Unmarshal([]byte(*got.UserData), &gotUD) + if err != nil { + t.Errorf("unable to unmarshal user data: %v", err) + } + if gotUD.BuildletBinaryURL != tc.hconf.BuildletBinaryURL(tc.buildEnv) { + t.Errorf("buildletBinaryURL got %s; want %s", gotUD.BuildletBinaryURL, tc.hconf.BuildletBinaryURL(tc.buildEnv)) + } + if gotUD.BuildletHostType != tc.hostType { + t.Errorf("buildletHostType got %s; want %s", gotUD.BuildletHostType, tc.hostType) + } + if gotUD.TLSCert != tc.opts.TLS.CertPEM { + t.Errorf("TLSCert got %s; want %s", gotUD.TLSCert, tc.opts.TLS.CertPEM) + } + if gotUD.TLSKey != tc.opts.TLS.KeyPEM { + t.Errorf("TLSKey got %s; want %s", gotUD.TLSKey, tc.opts.TLS.KeyPEM) + } + if gotUD.TLSPassword != tc.opts.TLS.Password() { + t.Errorf("TLSPassword got %s; want %s", gotUD.TLSPassword, tc.opts.TLS.Password()) + } + }) + } +} + +func TestEC2Instance(t *testing.T) { + instSample1 := &ec2.Instance{ + InstanceId: aws.String("id1"), + } + instSample2 := &ec2.Instance{ + InstanceId: aws.String("id2"), + } + resSample1 := &ec2.Reservation{ + Instances: []*ec2.Instance{ + instSample1, + }, + RequesterId: aws.String("user1"), + ReservationId: aws.String("reservation12"), + } + resSample2 := &ec2.Reservation{ + Instances: []*ec2.Instance{ + instSample2, + }, + RequesterId: aws.String("user2"), + ReservationId: aws.String("reservation22"), + } + + testCases := []struct { + desc string + dio *ec2.DescribeInstancesOutput + wantInst *ec2.Instance + }{ + { + desc: "single reservation", + dio: &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + resSample1, + }, + }, + wantInst: instSample1, + }, + { + desc: "multiple reservations", + dio: &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + resSample2, + resSample1, + }, + }, + wantInst: instSample2, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + gotInst, gotErr := ec2Instance(tc.dio) + if gotErr != nil { + t.Errorf("ec2Instance(%v) failed: %v", + tc.dio, gotErr) + } + if !cmp.Equal(gotInst, tc.wantInst) { + t.Errorf("ec2Instance(%v) = %s; want %s", + tc.dio, gotInst, tc.wantInst) + } + }) + } +} + +func TestEC2InstanceError(t *testing.T) { + testCases := []struct { + desc string + dio *ec2.DescribeInstancesOutput + }{ + { + desc: "nil input", + dio: nil, + }, + { + desc: "nil reservation", + dio: &ec2.DescribeInstancesOutput{ + Reservations: nil, + }, + }, + { + desc: "nil instances", + dio: &ec2.DescribeInstancesOutput{ + Reservations: []*ec2.Reservation{ + &ec2.Reservation{ + Instances: nil, + RequesterId: aws.String("user1"), + ReservationId: aws.String("reservation12"), + }, + }, + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + _, gotErr := ec2Instance(tc.dio) + if gotErr == nil { + t.Errorf("ec2Instance(%v) did not fail", tc.dio) + } + }) + } +} + +func TestEC2InstanceIPs(t *testing.T) { + testCases := []struct { + desc string + inst *ec2.Instance + wantIntIP string + wantExtIP string + }{ + { + desc: "base case", + inst: &ec2.Instance{ + PrivateIpAddress: aws.String("1.1.1.1"), + PublicIpAddress: aws.String("8.8.8.8"), + }, + wantIntIP: "1.1.1.1", + wantExtIP: "8.8.8.8", + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + gotIntIP, gotExtIP, gotErr := ec2InstanceIPs(tc.inst) + if gotErr != nil { + t.Errorf("ec2InstanceIPs(%v) failed: %v", + tc.inst, gotErr) + } + if gotIntIP != tc.wantIntIP || gotExtIP != tc.wantExtIP { + t.Errorf("ec2InstanceIPs(%v) = %s, %s, %v; want %s, %s, nil", + tc.inst, gotIntIP, gotExtIP, gotErr, tc.wantIntIP, tc.wantExtIP) + } + }) + } +} + +func TestEC2InstanceIPsErrors(t *testing.T) { + testCases := []struct { + desc string + inst *ec2.Instance + }{ + { + desc: "default vallues", + inst: &ec2.Instance{}, + }, + { + desc: "missing public ip", + inst: &ec2.Instance{ + PrivateIpAddress: aws.String("1.1.1.1"), + }, + }, + { + desc: "missing private ip", + inst: &ec2.Instance{ + PublicIpAddress: aws.String("8.8.8.8"), + }, + }, + { + desc: "empty public ip", + inst: &ec2.Instance{ + PrivateIpAddress: aws.String("1.1.1.1"), + PublicIpAddress: aws.String(""), + }, + }, + { + desc: "empty private ip", + inst: &ec2.Instance{ + PrivateIpAddress: aws.String(""), + PublicIpAddress: aws.String("8.8.8.8"), + }, + }, + } + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + _, _, gotErr := ec2InstanceIPs(tc.inst) + if gotErr == nil { + t.Errorf("ec2InstanceIPs(%v) = nil: want error", tc.inst) + } + }) + } +} diff --git a/buildlet/buildlet.go b/buildlet/buildlet.go new file mode 100644 index 0000000000..eb05a888b5 --- /dev/null +++ b/buildlet/buildlet.go @@ -0,0 +1,141 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildlet + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io/ioutil" + "net/http" + "time" + + "github.com/aws/aws-sdk-go/service/ec2" + "google.golang.org/api/compute/v1" +) + +// VMOpts control how new VMs are started. +type VMOpts struct { + // Zone is the GCE zone to create the VM in. + // Optional; defaults to provided build environment's zone. + Zone string + + // ProjectID is the GCE project ID (e.g. "foo-bar-123", not + // the numeric ID). + // Optional; defaults to provided build environment's project ID ("name"). + ProjectID string + + // TLS optionally specifies the TLS keypair to use. + // If zero, http without auth is used. + TLS KeyPair + + // Optional description of the VM. + Description string + + // Optional metadata to put on the instance. + Meta map[string]string + + // DeleteIn optionally specifies a duration at which + // to delete the VM. + // If zero, a reasonable default is used. + // Negative means no deletion timeout. + DeleteIn time.Duration + + // OnInstanceRequested optionally specifies a hook to run synchronously + // after the computeService.Instances.Insert call, but before + // waiting for its operation to proceed. + OnInstanceRequested func() + + // OnInstanceCreated optionally specifies a hook to run synchronously + // after the instance operation succeeds. + OnInstanceCreated func() + + // OnInstanceCreated optionally specifies a hook to run synchronously + // after the computeService.Instances.Get call. + // Only valid for GCE resources. + OnGotInstanceInfo func(*compute.Instance) + + // OnInstanceCreated optionally specifies a hook to run synchronously + // after the EC2 instance information is retrieved. + // Only valid for EC2 resources. + OnGotEC2InstanceInfo func(*ec2.Instance) + + // OnBeginBuildletProbe optionally specifies a hook to run synchronously + // before StartNewVM tries to hit buildletURL to see if it's up yet. + OnBeginBuildletProbe func(buildletURL string) + + // OnEndBuildletProbe optionally specifies a hook to run synchronously + // after StartNewVM tries to hit the buildlet's URL to see if it's up. + // The hook parameters are the return values from http.Get. + OnEndBuildletProbe func(*http.Response, error) + + // SkipEndpointVerification does not verify that the builder is listening + // on port 80 or 443 before creating a buildlet client. + SkipEndpointVerification bool +} + +// buildletClient returns a buildlet client configured to speak to a VM via the buildlet +// URL. The communication will use TLS if one is provided in the vmopts. This will wait until +// it can connect with the endpoint before returning. The buildletURL is in the form of: +// "https://". The ipPort field is in the form of ":". +func buildletClient(ctx context.Context, buildletURL, ipPort string, opts *VMOpts) (*Client, error) { + const timeout = 5 * time.Minute + deadline := time.Now().Add(timeout) + try := 0 + for !opts.SkipEndpointVerification { + try++ + if deadline.Before(time.Now()) { + return nil, fmt.Errorf("unable to probe buildet at %s in %v with %d attempts", buildletURL, timeout, try) + } + err := probeBuildlet(ctx, buildletURL, opts) + if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { + return nil, fmt.Errorf("unable to probe buildlet at %s: %w", buildletURL, err) + } + if err != nil { + time.Sleep(time.Second) + continue + } + break + } + return NewClient(ipPort, opts.TLS), nil +} + +// probeBuildlet attempts to the connect to a buildlet at the provided URL. An error +// is returned if it unable to connect to the buildlet. Each request is limited by either +// a five second limit or the timeout set in the context. +func probeBuildlet(ctx context.Context, buildletURL string, opts *VMOpts) error { + cl := &http.Client{ + Transport: &http.Transport{ + Dial: defaultDialer(), + DisableKeepAlives: true, + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + } + if fn := opts.OnBeginBuildletProbe; fn != nil { + fn(buildletURL) + } + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, buildletURL, nil) + if err != nil { + return fmt.Errorf("error creating buildlet probe request: %w", err) + } + res, err := cl.Do(req) + if fn := opts.OnEndBuildletProbe; fn != nil { + fn(res, err) + } + if err != nil { + return fmt.Errorf("error probe buildlet %s: %w", buildletURL, err) + } + ioutil.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != http.StatusOK { + return fmt.Errorf("buildlet returned HTTP status code %d for %s", res.StatusCode, buildletURL) + } + return nil +} diff --git a/buildlet/buildlet_test.go b/buildlet/buildlet_test.go new file mode 100644 index 0000000000..8bde7d3505 --- /dev/null +++ b/buildlet/buildlet_test.go @@ -0,0 +1,143 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package buildlet + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "testing" +) + +func TestBuildletClient(t *testing.T) { + var httpCalled, OnBeginBuildletProbeCalled, OnEndBuildletProbeCalled bool + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpCalled = true + fmt.Fprintln(w, "buildlet endpoint reached") + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("unable to parse http server url %s", err) + } + + kp, err := NewKeyPair() + if err != nil { + t.Fatalf("unable to create key pair %s", err) + } + + opt := &VMOpts{ + TLS: kp, + OnBeginBuildletProbe: func(string) { OnBeginBuildletProbeCalled = true }, + OnEndBuildletProbe: func(*http.Response, error) { OnEndBuildletProbeCalled = true }, + } + + gotClient, gotErr := buildletClient(context.Background(), ts.URL, u.Host, opt) + if gotErr != nil { + t.Errorf("buildletClient(ctx, %s, %s, %v) error %s", ts.URL, u.Host, opt, gotErr) + } + if gotClient == nil { + t.Errorf("client should not be nil") + } + if !httpCalled { + t.Error("http endpoint never called") + } + if !OnBeginBuildletProbeCalled { + t.Error("OnBeginBuildletProbe() was not called") + } + if !OnEndBuildletProbeCalled { + t.Error("OnEndBuildletProbe() was not called") + } +} + +func TestBuildletClientError(t *testing.T) { + var httpCalled, OnBeginBuildletProbeCalled, OnEndBuildletProbeCalled bool + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpCalled = true + fmt.Fprintln(w, "buildlet endpoint reached") + })) + defer ts.Close() + + u, err := url.Parse(ts.URL) + if err != nil { + t.Fatalf("unable to parse http server url %s", err) + } + + kp, err := NewKeyPair() + if err != nil { + t.Fatalf("unable to create key pair %s", err) + } + + opt := &VMOpts{ + TLS: kp, + OnBeginBuildletProbe: func(string) { OnBeginBuildletProbeCalled = true }, + OnEndBuildletProbe: func(*http.Response, error) { OnEndBuildletProbeCalled = true }, + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + gotClient, gotErr := buildletClient(ctx, ts.URL, u.Host, opt) + if gotErr == nil { + t.Errorf("buildletClient(ctx, %s, %s, %v) error %s", ts.URL, u.Host, opt, gotErr) + } + if gotClient != nil { + t.Errorf("client should be nil") + } + if httpCalled { + t.Error("http endpoint called") + } + if !OnBeginBuildletProbeCalled { + t.Error("OnBeginBuildletProbe() was not called") + } + if !OnEndBuildletProbeCalled { + t.Error("OnEndBuildletProbe() was not called") + } +} + +func TestProbeBuildlet(t *testing.T) { + var httpCalled, OnBeginBuildletProbeCalled, OnEndBuildletProbeCalled bool + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpCalled = true + fmt.Fprintln(w, "buildlet endpoint reached") + })) + defer ts.Close() + opt := &VMOpts{ + OnBeginBuildletProbe: func(string) { OnBeginBuildletProbeCalled = true }, + OnEndBuildletProbe: func(*http.Response, error) { OnEndBuildletProbeCalled = true }, + } + gotErr := probeBuildlet(context.Background(), ts.URL, opt) + if gotErr != nil { + t.Errorf("probeBuildlet(ctx, %q, %+v) = %s; want no error", ts.URL, opt, gotErr) + } + if !httpCalled { + t.Error("http endpoint never called") + } + if !OnBeginBuildletProbeCalled { + t.Error("OnBeginBuildletProbe() was not called") + } + if !OnEndBuildletProbeCalled { + t.Error("OnEndBuildletProbe() was not called") + } +} + +func TestProbeBuildletError(t *testing.T) { + var httpCalled bool + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + httpCalled = true + http.Error(w, "all types of broken", http.StatusInternalServerError) + })) + defer ts.Close() + opt := &VMOpts{} + gotErr := probeBuildlet(context.Background(), ts.URL, opt) + if gotErr == nil { + t.Errorf("probeBuildlet(ctx, %q, %+v) = nil; want error", ts.URL, opt) + } + if !httpCalled { + t.Error("http endpoint never called") + } +} diff --git a/buildlet/gce.go b/buildlet/gce.go index e31b39e1fc..04dcb31c0b 100644 --- a/buildlet/gce.go +++ b/buildlet/gce.go @@ -6,11 +6,9 @@ package buildlet import ( "context" - "crypto/tls" "errors" "fmt" "log" - "net/http" "sort" "strings" "sync" @@ -33,56 +31,6 @@ func apiGate() { } } -// VMOpts control how new VMs are started. -type VMOpts struct { - // Zone is the GCE zone to create the VM in. - // Optional; defaults to provided build environment's zone. - Zone string - - // ProjectID is the GCE project ID (e.g. "foo-bar-123", not - // the numeric ID). - // Optional; defaults to provided build environment's project ID ("name"). - ProjectID string - - // TLS optionally specifies the TLS keypair to use. - // If zero, http without auth is used. - TLS KeyPair - - // Optional description of the VM. - Description string - - // Optional metadata to put on the instance. - Meta map[string]string - - // DeleteIn optionally specifies a duration at which - // to delete the VM. - // If zero, a reasonable default is used. - // Negative means no deletion timeout. - DeleteIn time.Duration - - // OnInstanceRequested optionally specifies a hook to run synchronously - // after the computeService.Instances.Insert call, but before - // waiting for its operation to proceed. - OnInstanceRequested func() - - // OnInstanceCreated optionally specifies a hook to run synchronously - // after the instance operation succeeds. - OnInstanceCreated func() - - // OnInstanceCreated optionally specifies a hook to run synchronously - // after the computeService.Instances.Get call. - OnGotInstanceInfo func(*compute.Instance) - - // OnBeginBuildletProbe optionally specifies a hook to run synchronously - // before StartNewVM tries to hit buildletURL to see if it's up yet. - OnBeginBuildletProbe func(buildletURL string) - - // OnEndBuildletProbe optionally specifies a hook to run synchronously - // after StartNewVM tries to hit the buildlet's URL to see if it's up. - // The hook parameters are the return values from http.Get. - OnEndBuildletProbe func(*http.Response, error) -} - // StartNewVM boots a new VM on GCE and returns a buildlet client // configured to speak to it. func StartNewVM(creds *google.Credentials, buildEnv *buildenv.Environment, instName, hostType string, opts VMOpts) (*Client, error) { @@ -322,46 +270,7 @@ OpLoop: if opts.OnGotInstanceInfo != nil { opts.OnGotInstanceInfo(inst) } - - const timeout = 5 * time.Minute - var alive bool - impatientClient := &http.Client{ - Timeout: 5 * time.Second, - Transport: &http.Transport{ - Dial: defaultDialer(), - DisableKeepAlives: true, - TLSClientConfig: &tls.Config{ - InsecureSkipVerify: true, - }, - }, - } - deadline := time.Now().Add(timeout) - try := 0 - for time.Now().Before(deadline) { - try++ - if fn := opts.OnBeginBuildletProbe; fn != nil { - fn(buildletURL) - } - res, err := impatientClient.Get(buildletURL) - if fn := opts.OnEndBuildletProbe; fn != nil { - fn(res, err) - } - if err != nil { - time.Sleep(1 * time.Second) - continue - } - res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("buildlet returned HTTP status code %d on try number %d", res.StatusCode, try) - } - alive = true - break - } - if !alive { - return nil, fmt.Errorf("buildlet didn't come up at %s in %v", buildletURL, timeout) - } - - return NewClient(ipPort, opts.TLS), nil + return buildletClient(ctx, buildletURL, ipPort, &opts) } // DestroyVM sends a request to delete a VM. Actual VM description is diff --git a/dashboard/builders.go b/dashboard/builders.go index b900e25e3b..dce6bd7a30 100644 --- a/dashboard/builders.go +++ b/dashboard/builders.go @@ -551,6 +551,14 @@ var Hosts = map[string]*HostConfig{ env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"}, SSHUsername: "root", }, + "host-linux-arm64-aws": &HostConfig{ + Notes: "Debian Buster, EC2 arm64 instance. See x/build/env/linux-arm64/arm", + VMImage: "ami-0454a5239a73a9e81", + machineType: "a1.xlarge", + env: []string{"GOROOT_BOOTSTRAP=/usr/local/go-bootstrap"}, + buildletURLTmpl: "http://storage.googleapis.com/$BUCKET/buildlet.linux-amd64", + SSHUsername: "admin", + }, "host-illumos-amd64-jclulow": &HostConfig{ Notes: "SmartOS base64@19.1.0 zone", Owner: "josh@sysmgr.org", diff --git a/go.mod b/go.mod index 9d88bd5f59..22f943e274 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( cloud.google.com/go/storage v1.6.0 github.com/NYTimes/gziphandler v1.1.1 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 // indirect + github.com/aws/aws-sdk-go v1.30.15 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625 github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d github.com/davecgh/go-spew v1.1.1 diff --git a/go.sum b/go.sum index 2cbc718682..aa156707b0 100644 --- a/go.sum +++ b/go.sum @@ -39,6 +39,8 @@ github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cq github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/aws/aws-sdk-go v1.30.15 h1:Sd8QDVzzE8Sl+xNccmdj0HwMrFowv6uVUx9tGsCE1ZE= +github.com/aws/aws-sdk-go v1.30.15/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625 h1:ckJgFhFWywOx+YLEMIJsTb+NV6NexWICk5+AMSuz3ss= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -61,6 +63,7 @@ github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aev github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -122,6 +125,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1 h1:ujPKutqRlJtcfWk6toYVYagwra7HQHbXOaS171b4Tg8= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jmespath/go-jmespath v0.3.0 h1:OS12ieG61fsCg5+qLJ+SsW9NicxNkg3b25OyT2yCeUc= +github.com/jmespath/go-jmespath v0.3.0/go.mod h1:9QtRXoHjLGCJ5IBSaohpXITPlowMeeYCZ7fLUTSywik= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024 h1:rBMNdlhTLzJjJSDIjNEXX1Pz3Hmwmz91v+zycvx9PJc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1 h1:6QPYqodiu3GuPL+7mfx+NwDdp2eTkp9IfEUpgAwUN0o= @@ -134,6 +139,7 @@ github.com/kr/pty v1.1.3 h1:/Um6a/ZmD5tF7peoOJ5oN5KMQ0DrGVQSXLNwyckutPk= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -144,6 +150,8 @@ github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07 h1:UyzmZLoiDWMRywV4DUYb9Fbt8uiOSooupjTq10vpvnU= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= go.opencensus.io v0.21.0 h1:mU6zScU4U1YAFPHEHYk+3JC4SY7JxgkqS10ZOSyksNg=