Merge branch 'gruntwork-io:master' into upgrade-docker-compose-v2

go.mod

@@ -20,7 +20,7 @@ require (
 	github.com/google/go-containerregistry v0.6.0
 	github.com/google/uuid v1.2.0
 	github.com/gruntwork-io/go-commons v0.8.0
-	github.com/hashicorp/go-getter v1.5.9
+	github.com/hashicorp/go-getter v1.6.1
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-version v1.3.0
 	github.com/hashicorp/hcl/v2 v2.9.1
@@ -102,7 +102,7 @@ require (
 	golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 // indirect
 	golang.org/x/mod v0.4.2 // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
-	golang.org/x/sys v0.0.0-20210603125802-9665404d3644 // indirect
+	golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e // indirect
 	golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect

go.sum

@@ -482,8 +482,8 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv
 github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
 github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
 github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
-github.com/hashicorp/go-getter v1.5.9 h1:b7ahZW50iQiUek/at3CvZhPK1/jiV6CtKcsJiR6E4R0=
-github.com/hashicorp/go-getter v1.5.9/go.mod h1:BrrV/1clo8cCYu6mxvboYg+KutTiFnXjMEgDD8+i7ZI=
+github.com/hashicorp/go-getter v1.6.1 h1:NASsgP4q6tL94WH6nJxKWj8As2H/2kop/bB1d8JMyRY=
+github.com/hashicorp/go-getter v1.6.1/go.mod h1:IZCrswsZPeWv9IkVnLElzRU/gz/QPi6pZHn4tv6vbwA=
 github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
 github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
 github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I=
@@ -1069,8 +1069,9 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20210603125802-9665404d3644 h1:CA1DEQ4NdKphKeL70tvsWNdT5oFh1lOjihRcEDROi0I=
 golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e h1:w36l2Uw3dRan1K3TyXriXvY+6T56GNmlKGcqiQUJDfM=
+golang.org/x/sys v0.0.0-20220517195934-5e4e11fc645e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=

modules/aws/vpc.go

@@ -24,6 +24,7 @@ type Vpc struct {
 type Subnet struct {
 	Id               string            // The ID of the Subnet
 	AvailabilityZone string            // The Availability Zone the subnet is in
+	DefaultForAz     bool              // If the subnet is default for the Availability Zone
 	Tags             map[string]string // The tags associated with the subnet
 }
 
@@ -34,6 +35,7 @@ const vpcResourceTypeFilterValue = "vpc"
 const subnetResourceTypeFilterValue = "subnet"
 const isDefaultFilterName = "isDefault"
 const isDefaultFilterValue = "true"
+const defaultVPCName = "Default"
 
 // GetDefaultVpc fetches information about the default VPC in the given region.
 func GetDefaultVpc(t testing.TestingT, region string) *Vpc {
@@ -117,7 +119,7 @@ func FindVpcName(vpc *ec2.Vpc) string {
 	}
 
 	if *vpc.IsDefault {
-		return "Default"
+		return defaultVPCName
 	}
 
 	return ""
@@ -149,7 +151,7 @@ func GetSubnetsForVpcE(t testing.TestingT, vpcID string, region string) ([]Subne
 
 	for _, ec2Subnet := range subnetOutput.Subnets {
 		subnetTags := GetTagsForSubnet(t, *ec2Subnet.SubnetId, region)
-		subnet := Subnet{Id: aws.StringValue(ec2Subnet.SubnetId), AvailabilityZone: aws.StringValue(ec2Subnet.AvailabilityZone), Tags: subnetTags}
+		subnet := Subnet{Id: aws.StringValue(ec2Subnet.SubnetId), AvailabilityZone: aws.StringValue(ec2Subnet.AvailabilityZone), DefaultForAz: aws.BoolValue(ec2Subnet.DefaultForAz), Tags: subnetTags}
 		subnets = append(subnets, subnet)
 	}
 
@@ -182,6 +184,34 @@ func GetTagsForVpcE(t testing.TestingT, vpcID string, region string) (map[string
 	return tags, nil
 }
 
+// GetDefaultSubnetIDsForVpc gets the ids of the subnets that are the default subnet for the AvailabilityZone
+func GetDefaultSubnetIDsForVpc(t testing.TestingT, vpc Vpc) []string {
+	subnetIDs, err := GetDefaultSubnetIDsForVpcE(t, vpc)
+	require.NoError(t, err)
+	return subnetIDs
+}
+
+// GetDefaultSubnetIDsForVpcE gets the ids of the subnets that are the default subnet for the AvailabilityZone
+func GetDefaultSubnetIDsForVpcE(t testing.TestingT, vpc Vpc) ([]string, error) {
+	if vpc.Name != defaultVPCName {
+		// You cannot create a default subnet in a nondefault VPC
+		// https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html
+		return nil, fmt.Errorf("Only default VPCs have default subnets but VPC with id %s is not default VPC", vpc.Id)
+	}
+	subnetIDs := []string{}
+	numSubnets := len(vpc.Subnets)
+	if numSubnets == 0 {
+		return nil, fmt.Errorf("Expected to find at least one subnet in vpc with ID %s but found zero", vpc.Id)
+	}
+
+	for _, subnet := range vpc.Subnets {
+		if subnet.DefaultForAz {
+			subnetIDs = append(subnetIDs, subnet.Id)
+		}
+	}
+	return subnetIDs, nil
+}
+
 // GetTagsForSubnet gets the tags for the specified subnet.
 func GetTagsForSubnet(t testing.TestingT, subnetId string, region string) map[string]string {
 	tags, err := GetTagsForSubnetE(t, subnetId, region)
@@ -234,6 +264,14 @@ func IsPublicSubnetE(t testing.TestingT, subnetId string, region string) (bool,
 		return false, err
 	}
 
+	if len(rts.RouteTables) == 0 {
+		// Subnets not explicitly associated with any route table are implicitly associated with the main route table
+		rts, err = getImplicitRouteTableForSubnetE(t, subnetId, region)
+		if err != nil {
+			return false, err
+		}
+	}
+
 	for _, rt := range rts.RouteTables {
 		for _, r := range rt.Routes {
 			if strings.HasPrefix(aws.StringValue(r.GatewayId), "igw-") {
@@ -245,6 +283,40 @@ func IsPublicSubnetE(t testing.TestingT, subnetId string, region string) (bool,
 	return false, nil
 }
 
+func getImplicitRouteTableForSubnetE(t testing.TestingT, subnetId string, region string) (*ec2.DescribeRouteTablesOutput, error) {
+	mainRouteFilterName := "association.main"
+	mainRouteFilterValue := "true"
+	subnetFilterName := "subnet-id"
+
+	client, err := NewEc2ClientE(t, region)
+	if err != nil {
+		return nil, err
+	}
+
+	subnetFilter := ec2.Filter{
+		Name:   &subnetFilterName,
+		Values: []*string{&subnetId},
+	}
+	subnetOutput, err := client.DescribeSubnets(&ec2.DescribeSubnetsInput{Filters: []*ec2.Filter{&subnetFilter}})
+	if err != nil {
+		return nil, err
+	}
+	numSubnets := len(subnetOutput.Subnets)
+	if numSubnets != 1 {
+		return nil, fmt.Errorf("Expected to find one subnet with id %s but found %s", subnetId, strconv.Itoa(numSubnets))
+	}
+
+	mainRouteFilter := ec2.Filter{
+		Name:   &mainRouteFilterName,
+		Values: []*string{&mainRouteFilterValue},
+	}
+	vpcFilter := ec2.Filter{
+		Name:   aws.String(vpcIDFilterName),
+		Values: []*string{subnetOutput.Subnets[0].VpcId},
+	}
+	return client.DescribeRouteTables(&ec2.DescribeRouteTablesInput{Filters: []*ec2.Filter{&mainRouteFilter, &vpcFilter}})
+}
+
 // GetRandomPrivateCidrBlock gets a random CIDR block from the range of acceptable private IP addresses per RFC 1918
 // (https://tools.ietf.org/html/rfc1918#section-3)
 // The routingPrefix refers to the "/28" in 1.2.3.4/28.

modules/aws/vpc_test.go

@@ -1,6 +1,7 @@
 package aws
 
 import (
+	"fmt"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -49,7 +50,7 @@ func TestGetVpcsE(t *testing.T) {
 
 	// the default VPC has by default one subnet per availability zone
 	// https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html
-	assert.Equal(t, len(vpcs[0].Subnets), len(azs))
+	assert.True(t, len(vpcs[0].Subnets) >= len(azs))
 }
 
 func TestGetFirstTwoOctets(t *testing.T) {
@@ -76,6 +77,46 @@ func TestIsPublicSubnet(t *testing.T) {
 	assert.True(t, IsPublicSubnet(t, *subnet.SubnetId, region))
 }
 
+func TestGetDefaultSubnetIDsForVpc(t *testing.T) {
+	t.Parallel()
+
+	region := GetRandomStableRegion(t, nil, nil)
+	defaultVpcBeforeSubnetCreation := GetDefaultVpc(t, region)
+
+	// Creates a subnet in the default VPC with deferred deletion
+	// and fetches vpc object again
+	subnetName := fmt.Sprintf("%s-subnet", t.Name())
+	subnet := createPrivateSubnetInDefaultVpc(t, defaultVpcBeforeSubnetCreation.Id, subnetName, region)
+	defer deleteSubnet(t, *subnet.SubnetId, region)
+	defaultVpc := GetDefaultVpc(t, region)
+
+	defaultSubnetIDs := GetDefaultSubnetIDsForVpc(t, *defaultVpc)
+	assert.NotEmpty(t, defaultSubnetIDs)
+	// Checks that the amount of default subnets is smaller than
+	// total number of subnets in default vpc
+	assert.True(t, len(defaultSubnetIDs) < len(defaultVpc.Subnets))
+
+	availabilityZones := []string{}
+	for _, id := range defaultSubnetIDs {
+		// check if the recently created subnet does not come up here
+		assert.NotEqual(t, id, subnet.SubnetId)
+		// default subnets are by default public
+		// https://docs.aws.amazon.com/vpc/latest/userguide/default-vpc.html
+		assert.True(t, IsPublicSubnet(t, id, region))
+		for _, subnet := range defaultVpc.Subnets {
+			if id == subnet.Id {
+				availabilityZones = append(availabilityZones, subnet.AvailabilityZone)
+			}
+		}
+	}
+	// only one default subnet is allowed per AZ
+	uniqueAZs := map[string]bool{}
+	for _, az := range availabilityZones {
+		uniqueAZs[az] = true
+	}
+	assert.Equal(t, len(defaultSubnetIDs), len(uniqueAZs))
+}
+
 func TestGetTagsForVpc(t *testing.T) {
 	t.Parallel()
 
@@ -178,6 +219,38 @@ func createSubnet(t *testing.T, vpcId string, routeTableId string, region string
 	return *createSubnetOutput.Subnet
 }
 
+func createPrivateSubnetInDefaultVpc(t *testing.T, vpcId string, subnetName string, region string) ec2.Subnet {
+	ec2Client := NewEc2Client(t, region)
+
+	createSubnetOutput, err := ec2Client.CreateSubnet(&ec2.CreateSubnetInput{
+		CidrBlock: aws.String("172.31.172.0/24"),
+		VpcId:     aws.String(vpcId),
+		TagSpecifications: []*ec2.TagSpecification{
+			{
+				ResourceType: aws.String("subnet"),
+				Tags: []*ec2.Tag{
+					{
+						Key:   aws.String("Name"),
+						Value: aws.String(subnetName),
+					},
+				},
+			},
+		},
+	})
+	require.NoError(t, err)
+
+	return *createSubnetOutput.Subnet
+}
+
+func deleteSubnet(t *testing.T, subnetId string, region string) {
+	ec2Client := NewEc2Client(t, region)
+
+	_, err := ec2Client.DeleteSubnet(&ec2.DeleteSubnetInput{
+		SubnetId: aws.String(subnetId),
+	})
+	require.NoError(t, err)
+}
+
 func createVpc(t *testing.T, region string) ec2.Vpc {
 	ec2Client := NewEc2Client(t, region)
 

modules/gcp/compute_test.go

@@ -22,12 +22,15 @@ const DEFAULT_MACHINE_TYPE = "f1-micro"
 const DEFAULT_IMAGE_FAMILY_PROJECT_NAME = "ubuntu-os-cloud"
 const DEFAULT_IMAGE_FAMILY_NAME = "family/ubuntu-1804-lts"
 
+// Regions that don't support running f1-micro instances
+var RegionsToAvoid = []string{"asia-east2", "southamerica-west1", "europe-west8", "europe-southwest1"}
+
 func TestGetPublicIpOfInstance(t *testing.T) {
 	t.Parallel()
 
 	instanceName := RandomValidGcpName()
 	projectID := GetGoogleProjectIDFromEnvVar(t)
-	zone := GetRandomZone(t, projectID, nil, nil, []string{"southamerica-west1"})
+	zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid)
 
 	createComputeInstance(t, projectID, zone, instanceName)
 	defer deleteComputeInstance(t, projectID, zone, instanceName)
@@ -74,8 +77,7 @@ func TestGetAndSetLabels(t *testing.T) {
 	instanceName := RandomValidGcpName()
 	projectID := GetGoogleProjectIDFromEnvVar(t)
 
-	// On October 22, 2018, GCP launched the asia-east2 region, which promptly failed all our tests, so blacklist asia-east2.
-	zone := GetRandomZone(t, projectID, nil, nil, []string{"asia-east2"})
+	zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid)
 
 	createComputeInstance(t, projectID, zone, instanceName)
 	defer deleteComputeInstance(t, projectID, zone, instanceName)
@@ -111,9 +113,7 @@ func TestGetAndSetMetadata(t *testing.T) {
 	projectID := GetGoogleProjectIDFromEnvVar(t)
 	instanceName := RandomValidGcpName()
 
-	// The following zones do not have f1-micro instances available, so we avoid them
-	zonesToAvoid := []string{"asia-east2", "southamerica-west1"}
-	zone := GetRandomZone(t, projectID, nil, nil, zonesToAvoid)
+	zone := GetRandomZone(t, projectID, nil, nil, RegionsToAvoid)
 
 	// Create a new Compute Instance
 	createComputeInstance(t, projectID, zone, instanceName)

modules/k8s/job_test.go

@@ -145,7 +145,7 @@ spec:
     spec:
       containers:
       - name: pi
-        image: perl
+        image: "perl:5.34.1"
         command: ["perl",  "-Mbignum=bpi", "-wle", "print bpi(2000)"]
       restartPolicy: Never
   backoffLimit: 4

modules/terraform/count.go

@@ -22,6 +22,9 @@ const (
 	destroyRegexp           = `Destroy complete! Resources: (\d+) destroyed\.`
 	planWithChangesRegexp   = `(\033\[1m)?Plan:(\033\[0m)? (\d+) to add, (\d+) to change, (\d+) to destroy\.`
 	planWithNoChangesRegexp = `No changes\. (Infrastructure is up-to-date)|(Your infrastructure matches the configuration)\.`
+
+	// '.' doesn't match newline by default in go. We must instruct the regex to match it with the 's' flag.
+	planWithNoInfraChangesRegexp = `(?s)You can apply this plan.+without changing any real infrastructure`
 )
 
 const getResourceCountErrMessage = "Can't parse Terraform output"
@@ -48,6 +51,7 @@ func GetResourceCountE(t testing.TestingT, cmdout string) (*ResourceCount, error
 		{destroyRegexp, -1, -1, 1},
 		{planWithChangesRegexp, 3, 4, 5},
 		{planWithNoChangesRegexp, -1, -1, -1},
+		{planWithNoInfraChangesRegexp, -1, -1, -1},
 	}
 
 	for _, tc := range terraformCommandPatterns {

modules/terraform/show.go

@@ -28,3 +28,21 @@ func ShowE(t testing.TestingT, options *Options) (string, error) {
 	}
 	return RunTerraformCommandAndGetStdoutE(t, options, args...)
 }
+
+func ShowWithStruct(t testing.TestingT, options *Options) *PlanStruct {
+	out, err := ShowWithStructE(t, options)
+	require.NoError(t, err)
+	return out
+}
+
+func ShowWithStructE(t testing.TestingT, options *Options) (*PlanStruct, error) {
+	json, err := ShowE(t, options)
+	if err != nil {
+		return nil, err
+	}
+	planStruct, err := parsePlanJson(json)
+	if err != nil {
+		return nil, err
+	}
+	return planStruct, nil
+}

modules/terraform/show_test.go

@@ -38,3 +38,33 @@ func TestShowWithInlinePlan(t *testing.T) {
 	planJSON := Show(t, showOptions)
 	require.Contains(t, planJSON, "null_resource.test[0]")
 }
+
+func TestShowWithStructInlinePlan(t *testing.T) {
+	t.Parallel()
+
+	testFolder, err := files.CopyTerraformFolderToTemp("../../test/fixtures/terraform-basic-configuration", t.Name())
+	require.NoError(t, err)
+	planFilePath := filepath.Join(testFolder, "plan.out")
+
+	options := &Options{
+		TerraformDir: testFolder,
+		PlanFilePath: planFilePath,
+		Vars: map[string]interface{}{
+			"cnt": 1,
+		},
+	}
+
+	out := InitAndPlan(t, options)
+	require.Contains(t, out, fmt.Sprintf("Saved the plan to: %s", planFilePath))
+	require.FileExists(t, planFilePath, "Plan file was not saved to expected location:", planFilePath)
+
+	// show command does not accept Vars
+	showOptions := &Options{
+		TerraformDir: testFolder,
+		PlanFilePath: planFilePath,
+	}
+
+	// Test the JSON string
+	plan := ShowWithStruct(t, showOptions)
+	require.Contains(t, plan.ResourcePlannedValuesMap, "null_resource.test[0]")
+}